2026-02-20T21:33:36.429 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-20T21:33:36.444 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-20T21:33:36.524 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486 branch: wip-pdonnell-testing-20260219.182737-tentacle description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-snappy policy/none rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-stress-workunit-fast-diff} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '61486' kernel: branch: distro kdb: true sha1: distro ktype: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial no_nested_subset: false os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: wip-pdonnell-testing-20260219.182737-tentacle ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: none client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluestore block size: 96636764160 bluestore compression algorithm: snappy bluestore compression mode: aggressive bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f7870454149f22574086dd2ff55d050585136531 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: f7870454149f22574086dd2ff55d050585136531 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-pdonnell-testing-20260219.182737-tentacle sha1: f7870454149f22574086dd2ff55d050585136531 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 7161 sha1: f7870454149f22574086dd2ff55d050585136531 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-pdonnell-testing-20260219.182737-tentacle suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f7870454149f22574086dd2ff55d050585136531 targets: trial032.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPRXC6A0nz0Ujzi5yHHzygV4NFoxUEbgNW5j0qSKAWKxFkkbu/brPWlpCdqn6tyLIGv6fhrzujk2pJ6qXpMi07s= trial107.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG0Ox7QcjWoqAVtqdwVfCJ6pcSiRRJmVoXtIhl0//qNUjkwWwsjgx3aj2wx7whWZfyuqAMUwwncaoPTMGgKoBzo= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: image RBD_IMAGE_FEATURES: layering,exclusive-lock,object-map,fast-diff RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-20_16:15:01 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699 2026-02-20T21:33:36.524 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa; will attempt to use it 2026-02-20T21:33:36.525 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa/tasks 2026-02-20T21:33:36.525 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-20T21:33:36.526 INFO:teuthology.task.internal:Checking packages... 2026-02-20T21:33:36.526 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'f7870454149f22574086dd2ff55d050585136531' 2026-02-20T21:33:36.526 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-20T21:33:36.526 INFO:teuthology.packaging:ref: None 2026-02-20T21:33:36.526 INFO:teuthology.packaging:tag: None 2026-02-20T21:33:36.526 INFO:teuthology.packaging:branch: wip-pdonnell-testing-20260219.182737-tentacle 2026-02-20T21:33:36.526 INFO:teuthology.packaging:sha1: f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:33:36.526 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=wip-pdonnell-testing-20260219.182737-tentacle 2026-02-20T21:33:36.652 INFO:teuthology.task.internal:Found packages for ceph version 20.2.0-677-gf7870454-1jammy 2026-02-20T21:33:36.653 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-20T21:33:36.664 INFO:teuthology.task.internal:no buildpackages task found 2026-02-20T21:33:36.664 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-20T21:33:36.680 INFO:teuthology.task.internal:Saving configuration 2026-02-20T21:33:36.695 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-20T21:33:36.706 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-20T21:33:36.783 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial032.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-20 21:29:06.795708', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILeKar09XG01om9BBjGDabdBgqc5f+fbXfN3UWmfgRPZ'} 2026-02-20T21:33:36.839 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial107.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-20 21:29:06.795091', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHO2BEBUrOG4md5yBdqZ84CsWTVZGD4SEy5DD+sJj2Df'} 2026-02-20T21:33:36.839 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-20T21:33:36.849 INFO:teuthology.task.internal:roles: ubuntu@trial032.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-20T21:33:36.849 INFO:teuthology.task.internal:roles: ubuntu@trial107.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-20T21:33:36.850 INFO:teuthology.run_tasks:Running task console_log... 2026-02-20T21:33:36.981 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x712e1dffaac0>, signals=[15]) 2026-02-20T21:33:36.982 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-20T21:33:36.992 INFO:teuthology.task.internal:Opening connections... 2026-02-20T21:33:36.992 DEBUG:teuthology.task.internal:connecting to ubuntu@trial032.front.sepia.ceph.com 2026-02-20T21:33:36.993 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:33:37.059 DEBUG:teuthology.task.internal:connecting to ubuntu@trial107.front.sepia.ceph.com 2026-02-20T21:33:37.059 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial107.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:33:37.178 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-20T21:33:37.192 DEBUG:teuthology.orchestra.run.trial032:> uname -m 2026-02-20T21:33:37.200 INFO:teuthology.orchestra.run.trial032.stdout:x86_64 2026-02-20T21:33:37.200 DEBUG:teuthology.orchestra.run.trial032:> cat /etc/os-release 2026-02-20T21:33:37.248 INFO:teuthology.orchestra.run.trial032.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-20T21:33:37.248 INFO:teuthology.orchestra.run.trial032.stdout:NAME="Ubuntu" 2026-02-20T21:33:37.248 INFO:teuthology.orchestra.run.trial032.stdout:VERSION_ID="22.04" 2026-02-20T21:33:37.249 INFO:teuthology.orchestra.run.trial032.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-20T21:33:37.249 INFO:teuthology.orchestra.run.trial032.stdout:VERSION_CODENAME=jammy 2026-02-20T21:33:37.249 INFO:teuthology.orchestra.run.trial032.stdout:ID=ubuntu 2026-02-20T21:33:37.249 INFO:teuthology.orchestra.run.trial032.stdout:ID_LIKE=debian 2026-02-20T21:33:37.249 INFO:teuthology.orchestra.run.trial032.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-20T21:33:37.249 INFO:teuthology.orchestra.run.trial032.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-20T21:33:37.249 INFO:teuthology.orchestra.run.trial032.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-20T21:33:37.249 INFO:teuthology.orchestra.run.trial032.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-20T21:33:37.249 INFO:teuthology.orchestra.run.trial032.stdout:UBUNTU_CODENAME=jammy 2026-02-20T21:33:37.250 INFO:teuthology.lock.ops:Updating trial032.front.sepia.ceph.com on lock server 2026-02-20T21:33:37.327 DEBUG:teuthology.orchestra.run.trial107:> uname -m 2026-02-20T21:33:37.331 INFO:teuthology.orchestra.run.trial107.stdout:x86_64 2026-02-20T21:33:37.332 DEBUG:teuthology.orchestra.run.trial107:> cat /etc/os-release 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:NAME="Ubuntu" 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:VERSION_ID="22.04" 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:VERSION_CODENAME=jammy 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:ID=ubuntu 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:ID_LIKE=debian 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-20T21:33:37.378 INFO:teuthology.orchestra.run.trial107.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-20T21:33:37.379 INFO:teuthology.orchestra.run.trial107.stdout:UBUNTU_CODENAME=jammy 2026-02-20T21:33:37.379 INFO:teuthology.lock.ops:Updating trial107.front.sepia.ceph.com on lock server 2026-02-20T21:33:37.456 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-20T21:33:37.471 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-20T21:33:37.483 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-20T21:33:37.483 DEBUG:teuthology.orchestra.run.trial032:> test '!' -e /home/ubuntu/cephtest 2026-02-20T21:33:37.485 DEBUG:teuthology.orchestra.run.trial107:> test '!' -e /home/ubuntu/cephtest 2026-02-20T21:33:37.489 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-20T21:33:37.500 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-20T21:33:37.500 DEBUG:teuthology.orchestra.run.trial032:> test -z $(ls -A /var/lib/ceph) 2026-02-20T21:33:37.530 DEBUG:teuthology.orchestra.run.trial107:> test -z $(ls -A /var/lib/ceph) 2026-02-20T21:33:37.533 INFO:teuthology.orchestra.run.trial032.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-20T21:33:37.535 INFO:teuthology.orchestra.run.trial107.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-20T21:33:37.536 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-20T21:33:37.692 INFO:teuthology.run_tasks:Running task kernel... 2026-02-20T21:33:37.710 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-20T21:33:37.710 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-20T21:33:37.711 DEBUG:teuthology.orchestra.run.trial032:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-20T21:33:37.711 DEBUG:teuthology.orchestra.run.trial107:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-20T21:33:37.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:33:37.714 DEBUG:teuthology.orchestra.run.trial107:> uname -r 2026-02-20T21:33:37.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:33:37.715 DEBUG:teuthology.orchestra.run.trial032:> uname -r 2026-02-20T21:33:37.758 INFO:teuthology.orchestra.run.trial107.stdout:5.15.0-170-generic 2026-02-20T21:33:37.759 INFO:teuthology.task.kernel:Running kernel on trial107: 5.15.0-170-generic 2026-02-20T21:33:37.759 DEBUG:teuthology.orchestra.run.trial107:> sudo apt-get clean 2026-02-20T21:33:37.763 INFO:teuthology.orchestra.run.trial032.stdout:5.15.0-170-generic 2026-02-20T21:33:37.764 INFO:teuthology.task.kernel:Running kernel on trial032: 5.15.0-170-generic 2026-02-20T21:33:37.764 DEBUG:teuthology.orchestra.run.trial032:> sudo apt-get clean 2026-02-20T21:33:37.821 DEBUG:teuthology.orchestra.run.trial107:> sudo apt-get update 2026-02-20T21:33:37.822 DEBUG:teuthology.orchestra.run.trial032:> sudo apt-get update 2026-02-20T21:33:38.253 INFO:teuthology.orchestra.run.trial032.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-20T21:33:38.292 INFO:teuthology.orchestra.run.trial107.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-20T21:33:38.326 INFO:teuthology.orchestra.run.trial032.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-20T21:33:38.376 INFO:teuthology.orchestra.run.trial107.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-20T21:33:38.606 INFO:teuthology.orchestra.run.trial032.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-20T21:33:38.693 INFO:teuthology.orchestra.run.trial107.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-20T21:33:38.729 INFO:teuthology.orchestra.run.trial032.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-20T21:33:38.833 INFO:teuthology.orchestra.run.trial107.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-20T21:33:38.843 INFO:teuthology.orchestra.run.trial032.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3245 kB] 2026-02-20T21:33:38.964 INFO:teuthology.orchestra.run.trial107.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3245 kB] 2026-02-20T21:33:39.171 INFO:teuthology.orchestra.run.trial032.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [495 kB] 2026-02-20T21:33:39.180 INFO:teuthology.orchestra.run.trial032.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.1 kB] 2026-02-20T21:33:39.181 INFO:teuthology.orchestra.run.trial032.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-20T21:33:39.209 INFO:teuthology.orchestra.run.trial032.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-20T21:33:39.215 INFO:teuthology.orchestra.run.trial032.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [30.3 kB] 2026-02-20T21:33:39.216 INFO:teuthology.orchestra.run.trial032.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5310 kB] 2026-02-20T21:33:39.344 INFO:teuthology.orchestra.run.trial032.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1005 kB] 2026-02-20T21:33:39.348 INFO:teuthology.orchestra.run.trial107.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [495 kB] 2026-02-20T21:33:39.360 INFO:teuthology.orchestra.run.trial107.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.1 kB] 2026-02-20T21:33:39.360 INFO:teuthology.orchestra.run.trial107.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-20T21:33:39.363 INFO:teuthology.orchestra.run.trial032.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2983 kB] 2026-02-20T21:33:39.393 INFO:teuthology.orchestra.run.trial107.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-20T21:33:39.400 INFO:teuthology.orchestra.run.trial107.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [30.3 kB] 2026-02-20T21:33:39.401 INFO:teuthology.orchestra.run.trial107.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5310 kB] 2026-02-20T21:33:39.433 INFO:teuthology.orchestra.run.trial032.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/main Translation-en [427 kB] 2026-02-20T21:33:39.441 INFO:teuthology.orchestra.run.trial032.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-20T21:33:39.441 INFO:teuthology.orchestra.run.trial032.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1018 kB] 2026-02-20T21:33:39.467 INFO:teuthology.orchestra.run.trial032.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [22.7 kB] 2026-02-20T21:33:39.468 INFO:teuthology.orchestra.run.trial032.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [5147 kB] 2026-02-20T21:33:39.548 INFO:teuthology.orchestra.run.trial107.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1005 kB] 2026-02-20T21:33:39.572 INFO:teuthology.orchestra.run.trial107.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2983 kB] 2026-02-20T21:33:39.591 INFO:teuthology.orchestra.run.trial032.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy-security/restricted Translation-en [979 kB] 2026-02-20T21:33:39.653 INFO:teuthology.orchestra.run.trial107.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/main Translation-en [427 kB] 2026-02-20T21:33:39.664 INFO:teuthology.orchestra.run.trial107.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-20T21:33:39.664 INFO:teuthology.orchestra.run.trial107.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1018 kB] 2026-02-20T21:33:39.694 INFO:teuthology.orchestra.run.trial107.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [22.7 kB] 2026-02-20T21:33:39.695 INFO:teuthology.orchestra.run.trial107.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [5147 kB] 2026-02-20T21:33:39.840 INFO:teuthology.orchestra.run.trial107.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy-security/restricted Translation-en [979 kB] 2026-02-20T21:33:43.134 INFO:teuthology.orchestra.run.trial032.stdout:Fetched 22.6 MB in 3s (8527 kB/s) 2026-02-20T21:33:43.302 INFO:teuthology.orchestra.run.trial107.stdout:Fetched 22.6 MB in 3s (7985 kB/s) 2026-02-20T21:33:43.665 INFO:teuthology.orchestra.run.trial032.stdout:Reading package lists... 2026-02-20T21:33:43.679 DEBUG:teuthology.orchestra.run.trial032:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-20T21:33:43.713 INFO:teuthology.orchestra.run.trial032.stdout:Reading package lists... 2026-02-20T21:33:43.825 INFO:teuthology.orchestra.run.trial032.stdout:Building dependency tree... 2026-02-20T21:33:43.825 INFO:teuthology.orchestra.run.trial032.stdout:Reading state information... 2026-02-20T21:33:43.842 INFO:teuthology.orchestra.run.trial107.stdout:Reading package lists... 2026-02-20T21:33:43.855 DEBUG:teuthology.orchestra.run.trial107:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-20T21:33:43.888 INFO:teuthology.orchestra.run.trial107.stdout:Reading package lists... 2026-02-20T21:33:43.912 INFO:teuthology.orchestra.run.trial032.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-20T21:33:43.913 INFO:teuthology.orchestra.run.trial032.stdout:linux-image-generic set to manually installed. 2026-02-20T21:33:43.913 INFO:teuthology.orchestra.run.trial032.stdout:The following packages were automatically installed and are no longer required: 2026-02-20T21:33:43.913 INFO:teuthology.orchestra.run.trial032.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-20T21:33:43.913 INFO:teuthology.orchestra.run.trial032.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-20T21:33:43.913 INFO:teuthology.orchestra.run.trial032.stdout: sg3-utils-udev 2026-02-20T21:33:43.913 INFO:teuthology.orchestra.run.trial032.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-20T21:33:43.932 INFO:teuthology.orchestra.run.trial032.stdout:0 upgraded, 0 newly installed, 0 to remove and 13 not upgraded. 2026-02-20T21:33:43.934 DEBUG:teuthology.orchestra.run.trial032:> dpkg -s linux-image-generic 2026-02-20T21:33:43.986 INFO:teuthology.orchestra.run.trial032.stdout:Package: linux-image-generic 2026-02-20T21:33:43.986 INFO:teuthology.orchestra.run.trial032.stdout:Status: install ok installed 2026-02-20T21:33:43.986 INFO:teuthology.orchestra.run.trial032.stdout:Priority: optional 2026-02-20T21:33:43.986 INFO:teuthology.orchestra.run.trial032.stdout:Section: kernel 2026-02-20T21:33:43.986 INFO:teuthology.orchestra.run.trial032.stdout:Installed-Size: 21 2026-02-20T21:33:43.986 INFO:teuthology.orchestra.run.trial032.stdout:Maintainer: Ubuntu Kernel Team 2026-02-20T21:33:43.987 INFO:teuthology.orchestra.run.trial032.stdout:Architecture: amd64 2026-02-20T21:33:43.987 INFO:teuthology.orchestra.run.trial032.stdout:Source: linux-meta 2026-02-20T21:33:43.987 INFO:teuthology.orchestra.run.trial032.stdout:Version: 5.15.0.170.159 2026-02-20T21:33:43.987 INFO:teuthology.orchestra.run.trial032.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-20T21:33:43.987 INFO:teuthology.orchestra.run.trial032.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-20T21:33:43.987 INFO:teuthology.orchestra.run.trial032.stdout:Recommends: thermald 2026-02-20T21:33:43.987 INFO:teuthology.orchestra.run.trial032.stdout:Description: Generic Linux kernel image 2026-02-20T21:33:43.987 INFO:teuthology.orchestra.run.trial032.stdout: This package will always depend on the latest generic kernel image 2026-02-20T21:33:43.987 INFO:teuthology.orchestra.run.trial032.stdout: available. 2026-02-20T21:33:43.988 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-20T21:33:43.988 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-20T21:33:43.988 DEBUG:teuthology.orchestra.run.trial032:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-20T21:33:43.998 INFO:teuthology.orchestra.run.trial107.stdout:Building dependency tree... 2026-02-20T21:33:43.999 INFO:teuthology.orchestra.run.trial107.stdout:Reading state information... 2026-02-20T21:33:44.036 INFO:teuthology.orchestra.run.trial032.stdout:ttyS1 2026-02-20T21:33:44.055 DEBUG:teuthology.parallel:result is None 2026-02-20T21:33:44.085 INFO:teuthology.orchestra.run.trial107.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-20T21:33:44.085 INFO:teuthology.orchestra.run.trial107.stdout:linux-image-generic set to manually installed. 2026-02-20T21:33:44.086 INFO:teuthology.orchestra.run.trial107.stdout:The following packages were automatically installed and are no longer required: 2026-02-20T21:33:44.086 INFO:teuthology.orchestra.run.trial107.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-20T21:33:44.086 INFO:teuthology.orchestra.run.trial107.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-20T21:33:44.086 INFO:teuthology.orchestra.run.trial107.stdout: sg3-utils-udev 2026-02-20T21:33:44.086 INFO:teuthology.orchestra.run.trial107.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-20T21:33:44.105 INFO:teuthology.orchestra.run.trial107.stdout:0 upgraded, 0 newly installed, 0 to remove and 13 not upgraded. 2026-02-20T21:33:44.107 DEBUG:teuthology.orchestra.run.trial107:> dpkg -s linux-image-generic 2026-02-20T21:33:44.156 INFO:teuthology.orchestra.run.trial107.stdout:Package: linux-image-generic 2026-02-20T21:33:44.156 INFO:teuthology.orchestra.run.trial107.stdout:Status: install ok installed 2026-02-20T21:33:44.156 INFO:teuthology.orchestra.run.trial107.stdout:Priority: optional 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Section: kernel 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Installed-Size: 21 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Maintainer: Ubuntu Kernel Team 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Architecture: amd64 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Source: linux-meta 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Version: 5.15.0.170.159 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Recommends: thermald 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout:Description: Generic Linux kernel image 2026-02-20T21:33:44.157 INFO:teuthology.orchestra.run.trial107.stdout: This package will always depend on the latest generic kernel image 2026-02-20T21:33:44.158 INFO:teuthology.orchestra.run.trial107.stdout: available. 2026-02-20T21:33:44.158 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-20T21:33:44.158 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-20T21:33:44.158 DEBUG:teuthology.orchestra.run.trial107:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-20T21:33:44.207 INFO:teuthology.orchestra.run.trial107.stdout:ttyS1 2026-02-20T21:33:44.229 DEBUG:teuthology.parallel:result is None 2026-02-20T21:33:44.229 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-20T21:33:44.241 INFO:teuthology.task.internal:Creating test directory... 2026-02-20T21:33:44.241 DEBUG:teuthology.orchestra.run.trial032:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-20T21:33:44.243 DEBUG:teuthology.orchestra.run.trial107:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-20T21:33:44.274 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-20T21:33:44.286 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-20T21:33:44.297 INFO:teuthology.task.internal:Creating archive directory... 2026-02-20T21:33:44.298 DEBUG:teuthology.orchestra.run.trial032:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-20T21:33:44.300 DEBUG:teuthology.orchestra.run.trial107:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-20T21:33:44.328 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-20T21:33:44.341 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-20T21:33:44.341 DEBUG:teuthology.orchestra.run.trial032:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-20T21:33:44.346 DEBUG:teuthology.orchestra.run.trial107:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-20T21:33:44.353 INFO:teuthology.orchestra.run.trial032.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T21:33:44.360 INFO:teuthology.orchestra.run.trial032.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T21:33:44.371 INFO:teuthology.orchestra.run.trial107.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T21:33:44.377 INFO:teuthology.orchestra.run.trial107.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T21:33:44.379 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-20T21:33:44.390 INFO:teuthology.task.internal:Configuring sudo... 2026-02-20T21:33:44.391 DEBUG:teuthology.orchestra.run.trial032:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-20T21:33:44.406 DEBUG:teuthology.orchestra.run.trial107:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-20T21:33:44.431 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-20T21:33:44.447 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-20T21:33:44.447 DEBUG:teuthology.orchestra.run.trial032:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-20T21:33:44.458 DEBUG:teuthology.orchestra.run.trial107:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-20T21:33:44.474 DEBUG:teuthology.orchestra.run.trial032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:33:44.508 DEBUG:teuthology.orchestra.run.trial032:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:33:44.552 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:33:44.552 DEBUG:teuthology.orchestra.run.trial032:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-20T21:33:44.602 DEBUG:teuthology.orchestra.run.trial107:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:33:44.606 DEBUG:teuthology.orchestra.run.trial107:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:33:44.650 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:33:44.651 DEBUG:teuthology.orchestra.run.trial107:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-20T21:33:44.700 DEBUG:teuthology.orchestra.run.trial032:> sudo service rsyslog restart 2026-02-20T21:33:44.702 DEBUG:teuthology.orchestra.run.trial107:> sudo service rsyslog restart 2026-02-20T21:33:44.761 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-20T21:33:44.773 INFO:teuthology.task.internal:Starting timer... 2026-02-20T21:33:44.773 INFO:teuthology.run_tasks:Running task pcp... 2026-02-20T21:33:44.795 INFO:teuthology.run_tasks:Running task selinux... 2026-02-20T21:33:44.813 DEBUG:teuthology.task.selinux:Excluding trial032: OS 'ubuntu' does not support SELinux 2026-02-20T21:33:44.813 DEBUG:teuthology.task.selinux:Excluding trial107: OS 'ubuntu' does not support SELinux 2026-02-20T21:33:44.814 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-20T21:33:44.814 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2026-02-20T21:33:44.814 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-20T21:33:44.814 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-20T21:33:44.829 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-20T21:33:44.843 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-20T21:33:44.844 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial032.front.sepia.ceph.com,trial107.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-20T21:36:23.482 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial032.front.sepia.ceph.com'), Remote(name='ubuntu@trial107.front.sepia.ceph.com')] 2026-02-20T21:36:23.482 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial032.front.sepia.ceph.com' 2026-02-20T21:36:23.483 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial032.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:36:23.556 DEBUG:teuthology.orchestra.run.trial032:> true 2026-02-20T21:36:23.802 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial032.front.sepia.ceph.com' 2026-02-20T21:36:23.803 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial107.front.sepia.ceph.com' 2026-02-20T21:36:23.804 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial107.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:36:23.911 DEBUG:teuthology.orchestra.run.trial107:> true 2026-02-20T21:36:24.158 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial107.front.sepia.ceph.com' 2026-02-20T21:36:24.158 INFO:teuthology.run_tasks:Running task clock... 2026-02-20T21:36:24.175 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-20T21:36:24.175 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-20T21:36:24.176 DEBUG:teuthology.orchestra.run.trial032:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:36:24.178 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-20T21:36:24.178 DEBUG:teuthology.orchestra.run.trial107:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:36:24.196 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-20T21:36:24.196 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: Command line: ntpd -gq 2026-02-20T21:36:24.196 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: ---------------------------------------------------- 2026-02-20T21:36:24.196 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: ntp-4 is maintained by Network Time Foundation, 2026-02-20T21:36:24.196 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-20T21:36:24.196 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: corporation. Support and training for ntp-4 are 2026-02-20T21:36:24.197 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: available at https://www.nwtime.org/support 2026-02-20T21:36:24.197 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: ---------------------------------------------------- 2026-02-20T21:36:24.197 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: proto: precision = 0.030 usec (-25) 2026-02-20T21:36:24.197 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: basedate set to 2022-02-04 2026-02-20T21:36:24.197 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: gps base set to 2022-02-06 (week 2196) 2026-02-20T21:36:24.197 INFO:teuthology.orchestra.run.trial032.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-20T21:36:24.198 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-20T21:36:24.198 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: restrict ::: KOD does nothing without LIMITED. 2026-02-20T21:36:24.198 INFO:teuthology.orchestra.run.trial032.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-20T21:36:24.199 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: Listen and drop on 0 v6wildcard [::]:123 2026-02-20T21:36:24.199 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-20T21:36:24.199 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: Listen normally on 2 lo 127.0.0.1:123 2026-02-20T21:36:24.199 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: Listen normally on 3 enp1s0f0 10.20.193.32:123 2026-02-20T21:36:24.199 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: Listen normally on 4 lo [::1]:123 2026-02-20T21:36:24.200 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:622e%2]:123 2026-02-20T21:36:24.200 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:24 ntpd[9428]: Listening on routing socket on fd #22 for interface updates 2026-02-20T21:36:24.219 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-20T21:36:24.219 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: Command line: ntpd -gq 2026-02-20T21:36:24.219 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: ---------------------------------------------------- 2026-02-20T21:36:24.219 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: ntp-4 is maintained by Network Time Foundation, 2026-02-20T21:36:24.219 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-20T21:36:24.219 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: corporation. Support and training for ntp-4 are 2026-02-20T21:36:24.220 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: available at https://www.nwtime.org/support 2026-02-20T21:36:24.220 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: ---------------------------------------------------- 2026-02-20T21:36:24.220 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: proto: precision = 0.020 usec (-25) 2026-02-20T21:36:24.220 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: basedate set to 2022-02-04 2026-02-20T21:36:24.220 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: gps base set to 2022-02-06 (week 2196) 2026-02-20T21:36:24.220 INFO:teuthology.orchestra.run.trial107.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-20T21:36:24.221 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-20T21:36:24.221 INFO:teuthology.orchestra.run.trial107.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-20T21:36:24.222 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: restrict ::: KOD does nothing without LIMITED. 2026-02-20T21:36:24.222 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: Listen and drop on 0 v6wildcard [::]:123 2026-02-20T21:36:24.222 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-20T21:36:24.222 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: Listen normally on 2 lo 127.0.0.1:123 2026-02-20T21:36:24.222 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: Listen normally on 3 enp1s0f0 10.20.193.107:123 2026-02-20T21:36:24.222 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: Listen normally on 4 lo [::1]:123 2026-02-20T21:36:24.223 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:5e18%2]:123 2026-02-20T21:36:24.223 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:24 ntpd[9316]: Listening on routing socket on fd #22 for interface updates 2026-02-20T21:36:25.195 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:25 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:25.196 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:25 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:25.218 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:25.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:25.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:25.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:25.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:25.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:25.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:25.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:25.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:25.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:25.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:25 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:25.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:25 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:25.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:25 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:25.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:25 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:25.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:25.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:25.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:25.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:25.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:25.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:25.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:25.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:25 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:26.195 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:26 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:26.195 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:26 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:26.204 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:26 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:26.204 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:26 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:26.204 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:26 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:26.204 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:26 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:27.195 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:27 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:27.195 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:27 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:27.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:27.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:27.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:27.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:27.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:27.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:27.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:27.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:27.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:27.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:27.225 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:27 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:27.225 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:27 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:27.225 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:27 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:27.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:27 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:27.227 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:27.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:27.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:27.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:27.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:27.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:27.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:27.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:27 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:28.196 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:28 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:28.196 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:28 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:28.204 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:28 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:28.204 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:28 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:28.205 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:28 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:28.205 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:28 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:28.218 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:28 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:28.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:28 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:28.249 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:28 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:28.249 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:28 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:28.249 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:28 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:28.249 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:28 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:29.196 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.196 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.197 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.197 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.197 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.197 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.199 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.199 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.199 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:29.199 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:29.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:29.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:29.225 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:29.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:29.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:29.228 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:29.235 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.235 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.235 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:29.235 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:29 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:29.257 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:29.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:29.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:29.258 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:29 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:30.196 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:30 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:30.196 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:30 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:30.204 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:30 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:30.205 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:30 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:30.205 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:30 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:30.205 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:30 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:30.218 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:30 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:30.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:30 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:30.248 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:30 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:30.248 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:30 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:30.248 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:30 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:30.249 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:30 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:31.196 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:31.196 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:31.197 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:31.197 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:31.197 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:31.197 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:31.199 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:31.199 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:31.199 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:31.199 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:31.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:31.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:31.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:31.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:31.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:31.219 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:31.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:31.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:31.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:31.222 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:31.223 INFO:teuthology.orchestra.run.trial107.stdout:20 Feb 21:36:31 ntpd[9316]: ntpd: time slew -0.000990 s 2026-02-20T21:36:31.223 INFO:teuthology.orchestra.run.trial107.stdout:ntpd: time slew -0.000990s 2026-02-20T21:36:31.223 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-20T21:36:31.224 INFO:teuthology.orchestra.run.trial107.stderr:20 Feb 21:36:31 ntpd[9316]: can't open /var/log/ntpstats/loopstats.20260220: Permission denied 2026-02-20T21:36:31.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-20T21:36:31.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: can't open /var/log/ntpstats/rawstats.20260220: Permission denied 2026-02-20T21:36:31.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-20T21:36:31.226 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: can't open /var/log/ntpstats/peerstats.20260220: Permission denied 2026-02-20T21:36:31.226 INFO:teuthology.orchestra.run.trial032.stdout:20 Feb 21:36:31 ntpd[9428]: ntpd: time slew +0.002352 s 2026-02-20T21:36:31.226 INFO:teuthology.orchestra.run.trial032.stdout:ntpd: time slew +0.002352s 2026-02-20T21:36:31.227 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-20T21:36:31.227 INFO:teuthology.orchestra.run.trial032.stderr:20 Feb 21:36:31 ntpd[9428]: can't open /var/log/ntpstats/loopstats.20260220: Permission denied 2026-02-20T21:36:31.282 INFO:teuthology.orchestra.run.trial032.stdout: remote refid st t when poll reach delay offset jitter 2026-02-20T21:36:31.282 INFO:teuthology.orchestra.run.trial032.stdout:============================================================================== 2026-02-20T21:36:31.282 INFO:teuthology.orchestra.run.trial032.stdout: frumentum-zeus. .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-20T21:36:31.282 INFO:teuthology.orchestra.run.trial032.stdout: 45.83.234.123 ( .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-20T21:36:31.282 INFO:teuthology.orchestra.run.trial032.stdout: 23.157.160.168 .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-20T21:36:31.282 INFO:teuthology.orchestra.run.trial032.stdout: falkor.smatwebd .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-20T21:36:31.550 INFO:teuthology.orchestra.run.trial107.stdout: remote refid st t when poll reach delay offset jitter 2026-02-20T21:36:31.550 INFO:teuthology.orchestra.run.trial107.stdout:============================================================================== 2026-02-20T21:36:31.550 INFO:teuthology.orchestra.run.trial107.stdout: triton.ellipse. .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-20T21:36:31.550 INFO:teuthology.orchestra.run.trial107.stdout: 12.193.130.137 .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-20T21:36:31.550 INFO:teuthology.orchestra.run.trial107.stdout: s2-b.time.mci1. .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-20T21:36:31.551 INFO:teuthology.orchestra.run.trial107.stdout: ntp.maxhost.io .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-20T21:36:31.552 INFO:teuthology.run_tasks:Running task install... 2026-02-20T21:36:31.566 DEBUG:teuthology.task.install:project ceph 2026-02-20T21:36:31.566 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}} 2026-02-20T21:36:31.567 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'} 2026-02-20T21:36:31.567 INFO:teuthology.task.install:Using flavor: default 2026-02-20T21:36:31.575 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-20T21:36:31.575 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-20T21:36:31.576 DEBUG:teuthology.orchestra.run.trial032:> sudo apt-key list | grep Ceph 2026-02-20T21:36:31.576 DEBUG:teuthology.orchestra.run.trial107:> sudo apt-key list | grep Ceph 2026-02-20T21:36:31.623 INFO:teuthology.orchestra.run.trial032.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-20T21:36:31.645 INFO:teuthology.orchestra.run.trial107.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-20T21:36:31.651 INFO:teuthology.orchestra.run.trial032.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-20T21:36:31.651 INFO:teuthology.orchestra.run.trial032.stdout:uid [ unknown] Ceph.com (release key) 2026-02-20T21:36:31.652 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-20T21:36:31.653 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:36:31.675 INFO:teuthology.orchestra.run.trial107.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-20T21:36:31.675 INFO:teuthology.orchestra.run.trial107.stdout:uid [ unknown] Ceph.com (release key) 2026-02-20T21:36:31.676 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-20T21:36:31.677 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:36:31.855 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default/ 2026-02-20T21:36:31.855 INFO:teuthology.task.install.deb:Package version is 20.2.0-677-gf7870454-1jammy 2026-02-20T21:36:31.867 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default/ 2026-02-20T21:36:31.867 INFO:teuthology.task.install.deb:Package version is 20.2.0-677-gf7870454-1jammy 2026-02-20T21:36:32.009 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:36:32.009 DEBUG:teuthology.orchestra.run.trial032:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-20T21:36:32.018 DEBUG:teuthology.orchestra.run.trial032:> sudo apt-get update 2026-02-20T21:36:32.021 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:36:32.021 DEBUG:teuthology.orchestra.run.trial107:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-20T21:36:32.031 DEBUG:teuthology.orchestra.run.trial107:> sudo apt-get update 2026-02-20T21:36:32.171 INFO:teuthology.orchestra.run.trial032.stdout:Ign:1 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy InRelease 2026-02-20T21:36:32.176 INFO:teuthology.orchestra.run.trial107.stdout:Ign:1 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy InRelease 2026-02-20T21:36:32.183 INFO:teuthology.orchestra.run.trial032.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy Release [7680 B] 2026-02-20T21:36:32.187 INFO:teuthology.orchestra.run.trial107.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy Release [7680 B] 2026-02-20T21:36:32.195 INFO:teuthology.orchestra.run.trial032.stdout:Ign:3 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-20T21:36:32.199 INFO:teuthology.orchestra.run.trial107.stdout:Ign:3 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-20T21:36:32.206 INFO:teuthology.orchestra.run.trial032.stdout:Get:4 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.9 kB] 2026-02-20T21:36:32.210 INFO:teuthology.orchestra.run.trial107.stdout:Get:4 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.9 kB] 2026-02-20T21:36:32.525 INFO:teuthology.orchestra.run.trial107.stdout:Hit:5 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-20T21:36:32.528 INFO:teuthology.orchestra.run.trial032.stdout:Hit:5 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-20T21:36:32.608 INFO:teuthology.orchestra.run.trial107.stdout:Hit:6 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-20T21:36:32.611 INFO:teuthology.orchestra.run.trial032.stdout:Hit:6 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-20T21:36:32.696 INFO:teuthology.orchestra.run.trial107.stdout:Hit:7 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-20T21:36:32.699 INFO:teuthology.orchestra.run.trial032.stdout:Hit:7 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-20T21:36:32.785 INFO:teuthology.orchestra.run.trial107.stdout:Hit:8 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-20T21:36:32.788 INFO:teuthology.orchestra.run.trial032.stdout:Hit:8 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-20T21:36:32.926 INFO:teuthology.orchestra.run.trial107.stdout:Fetched 26.5 kB in 1s (36.0 kB/s) 2026-02-20T21:36:32.929 INFO:teuthology.orchestra.run.trial032.stdout:Fetched 26.5 kB in 1s (35.5 kB/s) 2026-02-20T21:36:33.439 INFO:teuthology.orchestra.run.trial107.stdout:Reading package lists... 2026-02-20T21:36:33.446 INFO:teuthology.orchestra.run.trial032.stdout:Reading package lists... 2026-02-20T21:36:33.452 DEBUG:teuthology.orchestra.run.trial107:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=20.2.0-677-gf7870454-1jammy cephadm=20.2.0-677-gf7870454-1jammy ceph-mds=20.2.0-677-gf7870454-1jammy ceph-mgr=20.2.0-677-gf7870454-1jammy ceph-common=20.2.0-677-gf7870454-1jammy ceph-fuse=20.2.0-677-gf7870454-1jammy ceph-test=20.2.0-677-gf7870454-1jammy ceph-volume=20.2.0-677-gf7870454-1jammy radosgw=20.2.0-677-gf7870454-1jammy python3-rados=20.2.0-677-gf7870454-1jammy python3-rgw=20.2.0-677-gf7870454-1jammy python3-cephfs=20.2.0-677-gf7870454-1jammy python3-rbd=20.2.0-677-gf7870454-1jammy libcephfs2=20.2.0-677-gf7870454-1jammy libcephfs-dev=20.2.0-677-gf7870454-1jammy librados2=20.2.0-677-gf7870454-1jammy librbd1=20.2.0-677-gf7870454-1jammy rbd-fuse=20.2.0-677-gf7870454-1jammy rbd-mirror=20.2.0-677-gf7870454-1jammy 2026-02-20T21:36:33.460 DEBUG:teuthology.orchestra.run.trial032:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=20.2.0-677-gf7870454-1jammy cephadm=20.2.0-677-gf7870454-1jammy ceph-mds=20.2.0-677-gf7870454-1jammy ceph-mgr=20.2.0-677-gf7870454-1jammy ceph-common=20.2.0-677-gf7870454-1jammy ceph-fuse=20.2.0-677-gf7870454-1jammy ceph-test=20.2.0-677-gf7870454-1jammy ceph-volume=20.2.0-677-gf7870454-1jammy radosgw=20.2.0-677-gf7870454-1jammy python3-rados=20.2.0-677-gf7870454-1jammy python3-rgw=20.2.0-677-gf7870454-1jammy python3-cephfs=20.2.0-677-gf7870454-1jammy python3-rbd=20.2.0-677-gf7870454-1jammy libcephfs2=20.2.0-677-gf7870454-1jammy libcephfs-dev=20.2.0-677-gf7870454-1jammy librados2=20.2.0-677-gf7870454-1jammy librbd1=20.2.0-677-gf7870454-1jammy rbd-fuse=20.2.0-677-gf7870454-1jammy rbd-mirror=20.2.0-677-gf7870454-1jammy 2026-02-20T21:36:33.484 INFO:teuthology.orchestra.run.trial107.stdout:Reading package lists... 2026-02-20T21:36:33.494 INFO:teuthology.orchestra.run.trial032.stdout:Reading package lists... 2026-02-20T21:36:33.591 INFO:teuthology.orchestra.run.trial107.stdout:Building dependency tree... 2026-02-20T21:36:33.591 INFO:teuthology.orchestra.run.trial107.stdout:Reading state information... 2026-02-20T21:36:33.603 INFO:teuthology.orchestra.run.trial032.stdout:Building dependency tree... 2026-02-20T21:36:33.604 INFO:teuthology.orchestra.run.trial032.stdout:Reading state information... 2026-02-20T21:36:33.680 INFO:teuthology.orchestra.run.trial107.stdout:The following packages were automatically installed and are no longer required: 2026-02-20T21:36:33.681 INFO:teuthology.orchestra.run.trial107.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-20T21:36:33.681 INFO:teuthology.orchestra.run.trial107.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-20T21:36:33.681 INFO:teuthology.orchestra.run.trial107.stdout: sg3-utils-udev 2026-02-20T21:36:33.681 INFO:teuthology.orchestra.run.trial107.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-20T21:36:33.681 INFO:teuthology.orchestra.run.trial107.stdout:The following additional packages will be installed: 2026-02-20T21:36:33.681 INFO:teuthology.orchestra.run.trial107.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-20T21:36:33.681 INFO:teuthology.orchestra.run.trial107.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-02-20T21:36:33.681 INFO:teuthology.orchestra.run.trial107.stdout: libcephfs-daemon libcephfs-proxy2 libdouble-conversion3 libfuse2 libjq1 2026-02-20T21:36:33.681 INFO:teuthology.orchestra.run.trial107.stdout: liblttng-ust1 libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: libqt5dbus5 libqt5network5 libradosstriper1 librdkafka1 librgw2 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli python-asyncssh-doc 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-asyncssh python3-cachetools python3-ceph-argparse 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-ceph-common python3-cheroot python3-cherrypy3 python3-google-auth 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-kubernetes python3-natsort python3-pluggy python3-portend 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-pytest python3-repoze.lru python3-requests-oauthlib python3-routes 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-tempora python3-threadpoolctl python3-toml python3-wcwidth 2026-02-20T21:36:33.682 INFO:teuthology.orchestra.run.trial107.stdout: python3-webob python3-websocket python3-zc.lockfile qttranslations5-l10n 2026-02-20T21:36:33.683 INFO:teuthology.orchestra.run.trial107.stdout: socat xmlstarlet 2026-02-20T21:36:33.683 INFO:teuthology.orchestra.run.trial107.stdout:Suggested packages: 2026-02-20T21:36:33.683 INFO:teuthology.orchestra.run.trial107.stdout: python3-influxdb liblua5.3-dev luarocks python-natsort-doc python-psutil-doc 2026-02-20T21:36:33.683 INFO:teuthology.orchestra.run.trial107.stdout: subversion python-pygments-doc ttf-bitstream-vera python3-paste python3-dap 2026-02-20T21:36:33.683 INFO:teuthology.orchestra.run.trial107.stdout: python-sklearn-doc ipython3 python-webob-doc 2026-02-20T21:36:33.683 INFO:teuthology.orchestra.run.trial107.stdout:Recommended packages: 2026-02-20T21:36:33.683 INFO:teuthology.orchestra.run.trial107.stdout: btrfs-tools 2026-02-20T21:36:33.694 INFO:teuthology.orchestra.run.trial032.stdout:The following packages were automatically installed and are no longer required: 2026-02-20T21:36:33.694 INFO:teuthology.orchestra.run.trial032.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-20T21:36:33.694 INFO:teuthology.orchestra.run.trial032.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-20T21:36:33.694 INFO:teuthology.orchestra.run.trial032.stdout: sg3-utils-udev 2026-02-20T21:36:33.694 INFO:teuthology.orchestra.run.trial032.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-20T21:36:33.694 INFO:teuthology.orchestra.run.trial032.stdout:The following additional packages will be installed: 2026-02-20T21:36:33.694 INFO:teuthology.orchestra.run.trial032.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: libcephfs-daemon libcephfs-proxy2 libdouble-conversion3 libfuse2 libjq1 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: liblttng-ust1 libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: libqt5dbus5 libqt5network5 libradosstriper1 librdkafka1 librgw2 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli python-asyncssh-doc 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: python3-asyncssh python3-cachetools python3-ceph-argparse 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: python3-ceph-common python3-cheroot python3-cherrypy3 python3-google-auth 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: python3-jaraco.functools python3-jaraco.text python3-joblib 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: python3-kubernetes python3-natsort python3-pluggy python3-portend 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: python3-pytest python3-repoze.lru python3-requests-oauthlib python3-routes 2026-02-20T21:36:33.695 INFO:teuthology.orchestra.run.trial032.stdout: python3-rsa python3-simplejson python3-sklearn python3-sklearn-lib 2026-02-20T21:36:33.696 INFO:teuthology.orchestra.run.trial032.stdout: python3-tempora python3-threadpoolctl python3-toml python3-wcwidth 2026-02-20T21:36:33.696 INFO:teuthology.orchestra.run.trial032.stdout: python3-webob python3-websocket python3-zc.lockfile qttranslations5-l10n 2026-02-20T21:36:33.696 INFO:teuthology.orchestra.run.trial032.stdout: socat xmlstarlet 2026-02-20T21:36:33.696 INFO:teuthology.orchestra.run.trial032.stdout:Suggested packages: 2026-02-20T21:36:33.696 INFO:teuthology.orchestra.run.trial032.stdout: python3-influxdb liblua5.3-dev luarocks python-natsort-doc python-psutil-doc 2026-02-20T21:36:33.696 INFO:teuthology.orchestra.run.trial032.stdout: subversion python-pygments-doc ttf-bitstream-vera python3-paste python3-dap 2026-02-20T21:36:33.696 INFO:teuthology.orchestra.run.trial032.stdout: python-sklearn-doc ipython3 python-webob-doc 2026-02-20T21:36:33.696 INFO:teuthology.orchestra.run.trial032.stdout:Recommended packages: 2026-02-20T21:36:33.696 INFO:teuthology.orchestra.run.trial032.stdout: btrfs-tools 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout:The following NEW packages will be installed: 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: libcephfs-daemon libcephfs-dev libcephfs-proxy2 libcephfs2 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 liboath0 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 librados2 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python3-asyncssh 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2026-02-20T21:36:33.715 INFO:teuthology.orchestra.run.trial107.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-20T21:36:33.716 INFO:teuthology.orchestra.run.trial107.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-20T21:36:33.716 INFO:teuthology.orchestra.run.trial107.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-natsort 2026-02-20T21:36:33.716 INFO:teuthology.orchestra.run.trial107.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2026-02-20T21:36:33.716 INFO:teuthology.orchestra.run.trial107.stdout: python3-pygments python3-pytest python3-rados python3-rbd python3-repoze.lru 2026-02-20T21:36:33.716 INFO:teuthology.orchestra.run.trial107.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2026-02-20T21:36:33.716 INFO:teuthology.orchestra.run.trial107.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-02-20T21:36:33.716 INFO:teuthology.orchestra.run.trial107.stdout: python3-threadpoolctl python3-toml python3-wcwidth python3-webob 2026-02-20T21:36:33.716 INFO:teuthology.orchestra.run.trial107.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2026-02-20T21:36:33.716 INFO:teuthology.orchestra.run.trial107.stdout: rbd-mirror socat xmlstarlet 2026-02-20T21:36:33.728 INFO:teuthology.orchestra.run.trial032.stdout:The following NEW packages will be installed: 2026-02-20T21:36:33.728 INFO:teuthology.orchestra.run.trial032.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-20T21:36:33.728 INFO:teuthology.orchestra.run.trial032.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-20T21:36:33.729 INFO:teuthology.orchestra.run.trial032.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-02-20T21:36:33.729 INFO:teuthology.orchestra.run.trial032.stdout: libcephfs-daemon libcephfs-dev libcephfs-proxy2 libcephfs2 2026-02-20T21:36:33.729 INFO:teuthology.orchestra.run.trial032.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 libnbd0 liboath0 2026-02-20T21:36:33.729 INFO:teuthology.orchestra.run.trial032.stdout: libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 librados2 2026-02-20T21:36:33.729 INFO:teuthology.orchestra.run.trial032.stdout: libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2026-02-20T21:36:33.730 INFO:teuthology.orchestra.run.trial032.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python3-asyncssh 2026-02-20T21:36:33.730 INFO:teuthology.orchestra.run.trial032.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2026-02-20T21:36:33.730 INFO:teuthology.orchestra.run.trial032.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-20T21:36:33.730 INFO:teuthology.orchestra.run.trial032.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-20T21:36:33.731 INFO:teuthology.orchestra.run.trial032.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-natsort 2026-02-20T21:36:33.731 INFO:teuthology.orchestra.run.trial032.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2026-02-20T21:36:33.731 INFO:teuthology.orchestra.run.trial032.stdout: python3-pygments python3-pytest python3-rados python3-rbd python3-repoze.lru 2026-02-20T21:36:33.731 INFO:teuthology.orchestra.run.trial032.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2026-02-20T21:36:33.731 INFO:teuthology.orchestra.run.trial032.stdout: python3-simplejson python3-sklearn python3-sklearn-lib python3-tempora 2026-02-20T21:36:33.731 INFO:teuthology.orchestra.run.trial032.stdout: python3-threadpoolctl python3-toml python3-wcwidth python3-webob 2026-02-20T21:36:33.731 INFO:teuthology.orchestra.run.trial032.stdout: python3-websocket python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2026-02-20T21:36:33.731 INFO:teuthology.orchestra.run.trial032.stdout: rbd-mirror socat xmlstarlet 2026-02-20T21:36:33.794 INFO:teuthology.orchestra.run.trial107.stdout:0 upgraded, 87 newly installed, 0 to remove and 13 not upgraded. 2026-02-20T21:36:33.795 INFO:teuthology.orchestra.run.trial107.stdout:Need to get 283 MB of archives. 2026-02-20T21:36:33.795 INFO:teuthology.orchestra.run.trial107.stdout:After this operation, 1130 MB of additional disk space will be used. 2026-02-20T21:36:33.795 INFO:teuthology.orchestra.run.trial107.stdout:Get:1 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 20.2.0-677-gf7870454-1jammy [3562 kB] 2026-02-20T21:36:33.810 INFO:teuthology.orchestra.run.trial032.stdout:0 upgraded, 87 newly installed, 0 to remove and 13 not upgraded. 2026-02-20T21:36:33.810 INFO:teuthology.orchestra.run.trial032.stdout:Need to get 283 MB of archives. 2026-02-20T21:36:33.810 INFO:teuthology.orchestra.run.trial032.stdout:After this operation, 1130 MB of additional disk space will be used. 2026-02-20T21:36:33.810 INFO:teuthology.orchestra.run.trial032.stdout:Get:1 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 20.2.0-677-gf7870454-1jammy [3562 kB] 2026-02-20T21:36:33.905 INFO:teuthology.orchestra.run.trial107.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 20.2.0-677-gf7870454-1jammy [2867 kB] 2026-02-20T21:36:33.968 INFO:teuthology.orchestra.run.trial107.stdout:Get:3 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 20.2.0-677-gf7870454-1jammy [833 kB] 2026-02-20T21:36:33.982 INFO:teuthology.orchestra.run.trial107.stdout:Get:4 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 20.2.0-677-gf7870454-1jammy [364 kB] 2026-02-20T21:36:33.990 INFO:teuthology.orchestra.run.trial107.stdout:Get:5 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 20.2.0-677-gf7870454-1jammy [32.8 kB] 2026-02-20T21:36:33.990 INFO:teuthology.orchestra.run.trial107.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 20.2.0-677-gf7870454-1jammy [184 kB] 2026-02-20T21:36:33.992 INFO:teuthology.orchestra.run.trial107.stdout:Get:7 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 20.2.0-677-gf7870454-1jammy [83.9 kB] 2026-02-20T21:36:33.994 INFO:teuthology.orchestra.run.trial107.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 20.2.0-677-gf7870454-1jammy [341 kB] 2026-02-20T21:36:34.011 INFO:teuthology.orchestra.run.trial107.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 20.2.0-677-gf7870454-1jammy [8698 kB] 2026-02-20T21:36:34.070 INFO:teuthology.orchestra.run.trial107.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-20T21:36:34.076 INFO:teuthology.orchestra.run.trial032.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 20.2.0-677-gf7870454-1jammy [2867 kB] 2026-02-20T21:36:34.145 INFO:teuthology.orchestra.run.trial032.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-20T21:36:34.187 INFO:teuthology.orchestra.run.trial107.stdout:Get:11 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 20.2.0-677-gf7870454-1jammy [112 kB] 2026-02-20T21:36:34.188 INFO:teuthology.orchestra.run.trial107.stdout:Get:12 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 20.2.0-677-gf7870454-1jammy [261 kB] 2026-02-20T21:36:34.191 INFO:teuthology.orchestra.run.trial107.stdout:Get:13 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 20.2.0-677-gf7870454-1jammy [29.3 MB] 2026-02-20T21:36:34.262 INFO:teuthology.orchestra.run.trial032.stdout:Get:4 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 20.2.0-677-gf7870454-1jammy [833 kB] 2026-02-20T21:36:34.276 INFO:teuthology.orchestra.run.trial032.stdout:Get:5 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 20.2.0-677-gf7870454-1jammy [364 kB] 2026-02-20T21:36:34.332 INFO:teuthology.orchestra.run.trial032.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 20.2.0-677-gf7870454-1jammy [32.8 kB] 2026-02-20T21:36:34.333 INFO:teuthology.orchestra.run.trial032.stdout:Get:7 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 20.2.0-677-gf7870454-1jammy [184 kB] 2026-02-20T21:36:34.336 INFO:teuthology.orchestra.run.trial032.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 20.2.0-677-gf7870454-1jammy [83.9 kB] 2026-02-20T21:36:34.338 INFO:teuthology.orchestra.run.trial032.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 20.2.0-677-gf7870454-1jammy [341 kB] 2026-02-20T21:36:34.344 INFO:teuthology.orchestra.run.trial032.stdout:Get:10 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 20.2.0-677-gf7870454-1jammy [8698 kB] 2026-02-20T21:36:34.408 INFO:teuthology.orchestra.run.trial107.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-20T21:36:34.418 INFO:teuthology.orchestra.run.trial107.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-20T21:36:34.488 INFO:teuthology.orchestra.run.trial107.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-20T21:36:34.556 INFO:teuthology.orchestra.run.trial032.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-20T21:36:34.570 INFO:teuthology.orchestra.run.trial032.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-20T21:36:34.654 INFO:teuthology.orchestra.run.trial032.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-20T21:36:34.689 INFO:teuthology.orchestra.run.trial107.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-20T21:36:34.699 INFO:teuthology.orchestra.run.trial107.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-20T21:36:34.726 INFO:teuthology.orchestra.run.trial107.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-20T21:36:34.734 INFO:teuthology.orchestra.run.trial107.stdout:Get:20 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-20T21:36:34.736 INFO:teuthology.orchestra.run.trial107.stdout:Get:21 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-20T21:36:34.737 INFO:teuthology.orchestra.run.trial107.stdout:Get:22 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-20T21:36:34.737 INFO:teuthology.orchestra.run.trial107.stdout:Get:23 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-20T21:36:34.754 INFO:teuthology.orchestra.run.trial107.stdout:Get:24 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-20T21:36:34.755 INFO:teuthology.orchestra.run.trial107.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-20T21:36:34.761 INFO:teuthology.orchestra.run.trial107.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-20T21:36:34.794 INFO:teuthology.orchestra.run.trial032.stdout:Get:14 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 20.2.0-677-gf7870454-1jammy [112 kB] 2026-02-20T21:36:34.809 INFO:teuthology.orchestra.run.trial032.stdout:Get:15 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 20.2.0-677-gf7870454-1jammy [261 kB] 2026-02-20T21:36:34.813 INFO:teuthology.orchestra.run.trial032.stdout:Get:16 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 20.2.0-677-gf7870454-1jammy [29.3 MB] 2026-02-20T21:36:34.828 INFO:teuthology.orchestra.run.trial107.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-20T21:36:34.828 INFO:teuthology.orchestra.run.trial107.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-20T21:36:34.829 INFO:teuthology.orchestra.run.trial107.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-20T21:36:34.829 INFO:teuthology.orchestra.run.trial107.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-20T21:36:34.829 INFO:teuthology.orchestra.run.trial107.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-20T21:36:34.829 INFO:teuthology.orchestra.run.trial107.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-20T21:36:34.829 INFO:teuthology.orchestra.run.trial107.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-20T21:36:34.833 INFO:teuthology.orchestra.run.trial107.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-20T21:36:34.834 INFO:teuthology.orchestra.run.trial107.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-20T21:36:34.895 INFO:teuthology.orchestra.run.trial107.stdout:Get:36 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 20.2.0-677-gf7870454-1jammy [5416 kB] 2026-02-20T21:36:34.899 INFO:teuthology.orchestra.run.trial032.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-20T21:36:34.903 INFO:teuthology.orchestra.run.trial107.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-20T21:36:34.906 INFO:teuthology.orchestra.run.trial107.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-20T21:36:34.906 INFO:teuthology.orchestra.run.trial107.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-20T21:36:34.907 INFO:teuthology.orchestra.run.trial107.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-20T21:36:34.912 INFO:teuthology.orchestra.run.trial032.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-20T21:36:34.932 INFO:teuthology.orchestra.run.trial107.stdout:Get:41 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-20T21:36:34.934 INFO:teuthology.orchestra.run.trial107.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-20T21:36:34.934 INFO:teuthology.orchestra.run.trial107.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-20T21:36:34.944 INFO:teuthology.orchestra.run.trial032.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-20T21:36:34.954 INFO:teuthology.orchestra.run.trial032.stdout:Get:20 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-20T21:36:34.957 INFO:teuthology.orchestra.run.trial032.stdout:Get:21 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-20T21:36:34.958 INFO:teuthology.orchestra.run.trial032.stdout:Get:22 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-20T21:36:34.958 INFO:teuthology.orchestra.run.trial032.stdout:Get:23 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-20T21:36:34.979 INFO:teuthology.orchestra.run.trial032.stdout:Get:24 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-20T21:36:34.980 INFO:teuthology.orchestra.run.trial032.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-20T21:36:34.987 INFO:teuthology.orchestra.run.trial032.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-20T21:36:34.991 INFO:teuthology.orchestra.run.trial107.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-20T21:36:34.992 INFO:teuthology.orchestra.run.trial107.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-20T21:36:34.992 INFO:teuthology.orchestra.run.trial107.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-20T21:36:34.993 INFO:teuthology.orchestra.run.trial107.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-20T21:36:34.993 INFO:teuthology.orchestra.run.trial107.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-20T21:36:34.993 INFO:teuthology.orchestra.run.trial107.stdout:Get:49 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-20T21:36:35.009 INFO:teuthology.orchestra.run.trial107.stdout:Get:50 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 20.2.0-677-gf7870454-1jammy [246 kB] 2026-02-20T21:36:35.016 INFO:teuthology.orchestra.run.trial107.stdout:Get:51 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 20.2.0-677-gf7870454-1jammy [124 kB] 2026-02-20T21:36:35.018 INFO:teuthology.orchestra.run.trial107.stdout:Get:52 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 20.2.0-677-gf7870454-1jammy [906 kB] 2026-02-20T21:36:35.046 INFO:teuthology.orchestra.run.trial107.stdout:Get:53 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 20.2.0-677-gf7870454-1jammy [6401 kB] 2026-02-20T21:36:35.052 INFO:teuthology.orchestra.run.trial107.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-20T21:36:35.054 INFO:teuthology.orchestra.run.trial107.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-20T21:36:35.055 INFO:teuthology.orchestra.run.trial107.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-20T21:36:35.068 INFO:teuthology.orchestra.run.trial032.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-20T21:36:35.069 INFO:teuthology.orchestra.run.trial032.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-20T21:36:35.069 INFO:teuthology.orchestra.run.trial032.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-20T21:36:35.069 INFO:teuthology.orchestra.run.trial032.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-20T21:36:35.069 INFO:teuthology.orchestra.run.trial032.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-20T21:36:35.070 INFO:teuthology.orchestra.run.trial032.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-20T21:36:35.070 INFO:teuthology.orchestra.run.trial032.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-20T21:36:35.075 INFO:teuthology.orchestra.run.trial032.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-20T21:36:35.076 INFO:teuthology.orchestra.run.trial032.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-20T21:36:35.124 INFO:teuthology.orchestra.run.trial107.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-20T21:36:35.127 INFO:teuthology.orchestra.run.trial107.stdout:Get:58 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-20T21:36:35.129 INFO:teuthology.orchestra.run.trial107.stdout:Get:59 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-20T21:36:35.133 INFO:teuthology.orchestra.run.trial107.stdout:Get:60 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-20T21:36:35.136 INFO:teuthology.orchestra.run.trial107.stdout:Get:61 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-20T21:36:35.136 INFO:teuthology.orchestra.run.trial107.stdout:Get:62 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-20T21:36:35.136 INFO:teuthology.orchestra.run.trial107.stdout:Get:63 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-20T21:36:35.160 INFO:teuthology.orchestra.run.trial032.stdout:Get:36 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-20T21:36:35.163 INFO:teuthology.orchestra.run.trial032.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-20T21:36:35.163 INFO:teuthology.orchestra.run.trial032.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-20T21:36:35.164 INFO:teuthology.orchestra.run.trial032.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-20T21:36:35.186 INFO:teuthology.orchestra.run.trial107.stdout:Get:64 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 20.2.0-677-gf7870454-1jammy [21.7 MB] 2026-02-20T21:36:35.195 INFO:teuthology.orchestra.run.trial032.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-20T21:36:35.196 INFO:teuthology.orchestra.run.trial107.stdout:Get:65 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-20T21:36:35.197 INFO:teuthology.orchestra.run.trial032.stdout:Get:41 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-20T21:36:35.197 INFO:teuthology.orchestra.run.trial107.stdout:Get:66 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-20T21:36:35.198 INFO:teuthology.orchestra.run.trial032.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-20T21:36:35.203 INFO:teuthology.orchestra.run.trial107.stdout:Get:67 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-20T21:36:35.268 INFO:teuthology.orchestra.run.trial032.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-20T21:36:35.269 INFO:teuthology.orchestra.run.trial107.stdout:Get:68 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-20T21:36:35.269 INFO:teuthology.orchestra.run.trial032.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-20T21:36:35.269 INFO:teuthology.orchestra.run.trial032.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-20T21:36:35.270 INFO:teuthology.orchestra.run.trial032.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-20T21:36:35.270 INFO:teuthology.orchestra.run.trial032.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-20T21:36:35.270 INFO:teuthology.orchestra.run.trial032.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-20T21:36:35.271 INFO:teuthology.orchestra.run.trial107.stdout:Get:69 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-20T21:36:35.271 INFO:teuthology.orchestra.run.trial107.stdout:Get:70 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-20T21:36:35.272 INFO:teuthology.orchestra.run.trial107.stdout:Get:71 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-20T21:36:35.342 INFO:teuthology.orchestra.run.trial032.stdout:Get:49 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-20T21:36:35.343 INFO:teuthology.orchestra.run.trial032.stdout:Get:50 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-20T21:36:35.345 INFO:teuthology.orchestra.run.trial032.stdout:Get:51 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-20T21:36:35.430 INFO:teuthology.orchestra.run.trial032.stdout:Get:52 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-20T21:36:35.433 INFO:teuthology.orchestra.run.trial032.stdout:Get:53 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-20T21:36:35.436 INFO:teuthology.orchestra.run.trial032.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-20T21:36:35.441 INFO:teuthology.orchestra.run.trial032.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-20T21:36:35.444 INFO:teuthology.orchestra.run.trial032.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-20T21:36:35.444 INFO:teuthology.orchestra.run.trial032.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-20T21:36:35.444 INFO:teuthology.orchestra.run.trial032.stdout:Get:58 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-20T21:36:35.518 INFO:teuthology.orchestra.run.trial032.stdout:Get:59 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-20T21:36:35.519 INFO:teuthology.orchestra.run.trial032.stdout:Get:60 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-20T21:36:35.526 INFO:teuthology.orchestra.run.trial032.stdout:Get:61 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-20T21:36:35.606 INFO:teuthology.orchestra.run.trial032.stdout:Get:62 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-20T21:36:35.608 INFO:teuthology.orchestra.run.trial032.stdout:Get:63 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-20T21:36:35.609 INFO:teuthology.orchestra.run.trial032.stdout:Get:64 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-20T21:36:35.609 INFO:teuthology.orchestra.run.trial032.stdout:Get:65 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-20T21:36:35.687 INFO:teuthology.orchestra.run.trial107.stdout:Get:72 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 20.2.0-677-gf7870454-1jammy [14.1 kB] 2026-02-20T21:36:35.688 INFO:teuthology.orchestra.run.trial107.stdout:Get:73 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 20.2.0-677-gf7870454-1jammy [957 kB] 2026-02-20T21:36:35.704 INFO:teuthology.orchestra.run.trial107.stdout:Get:74 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 20.2.0-677-gf7870454-1jammy [2345 kB] 2026-02-20T21:36:35.759 INFO:teuthology.orchestra.run.trial107.stdout:Get:75 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 20.2.0-677-gf7870454-1jammy [1049 kB] 2026-02-20T21:36:35.790 INFO:teuthology.orchestra.run.trial107.stdout:Get:76 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 20.2.0-677-gf7870454-1jammy [179 kB] 2026-02-20T21:36:35.793 INFO:teuthology.orchestra.run.trial107.stdout:Get:77 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 20.2.0-677-gf7870454-1jammy [45.5 MB] 2026-02-20T21:36:36.507 INFO:teuthology.orchestra.run.trial032.stdout:Get:66 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 20.2.0-677-gf7870454-1jammy [5416 kB] 2026-02-20T21:36:36.804 INFO:teuthology.orchestra.run.trial032.stdout:Get:67 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 20.2.0-677-gf7870454-1jammy [246 kB] 2026-02-20T21:36:36.826 INFO:teuthology.orchestra.run.trial032.stdout:Get:68 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 20.2.0-677-gf7870454-1jammy [124 kB] 2026-02-20T21:36:36.828 INFO:teuthology.orchestra.run.trial032.stdout:Get:69 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 20.2.0-677-gf7870454-1jammy [906 kB] 2026-02-20T21:36:36.848 INFO:teuthology.orchestra.run.trial107.stdout:Get:78 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 20.2.0-677-gf7870454-1jammy [8625 kB] 2026-02-20T21:36:36.887 INFO:teuthology.orchestra.run.trial032.stdout:Get:70 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 20.2.0-677-gf7870454-1jammy [6401 kB] 2026-02-20T21:36:37.037 INFO:teuthology.orchestra.run.trial107.stdout:Get:79 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 20.2.0-677-gf7870454-1jammy [14.2 kB] 2026-02-20T21:36:37.038 INFO:teuthology.orchestra.run.trial107.stdout:Get:80 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 20.2.0-677-gf7870454-1jammy [99.5 MB] 2026-02-20T21:36:37.224 INFO:teuthology.orchestra.run.trial032.stdout:Get:71 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 20.2.0-677-gf7870454-1jammy [21.7 MB] 2026-02-20T21:36:38.466 INFO:teuthology.orchestra.run.trial032.stdout:Get:72 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 20.2.0-677-gf7870454-1jammy [14.1 kB] 2026-02-20T21:36:38.466 INFO:teuthology.orchestra.run.trial032.stdout:Get:73 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 20.2.0-677-gf7870454-1jammy [957 kB] 2026-02-20T21:36:38.499 INFO:teuthology.orchestra.run.trial032.stdout:Get:74 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 20.2.0-677-gf7870454-1jammy [2345 kB] 2026-02-20T21:36:38.636 INFO:teuthology.orchestra.run.trial032.stdout:Get:75 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 20.2.0-677-gf7870454-1jammy [1049 kB] 2026-02-20T21:36:38.702 INFO:teuthology.orchestra.run.trial032.stdout:Get:76 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 20.2.0-677-gf7870454-1jammy [179 kB] 2026-02-20T21:36:38.705 INFO:teuthology.orchestra.run.trial032.stdout:Get:77 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 20.2.0-677-gf7870454-1jammy [45.5 MB] 2026-02-20T21:36:39.348 INFO:teuthology.orchestra.run.trial107.stdout:Get:81 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 20.2.0-677-gf7870454-1jammy [135 kB] 2026-02-20T21:36:39.348 INFO:teuthology.orchestra.run.trial107.stdout:Get:82 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-daemon amd64 20.2.0-677-gf7870454-1jammy [43.3 kB] 2026-02-20T21:36:39.349 INFO:teuthology.orchestra.run.trial107.stdout:Get:83 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-proxy2 amd64 20.2.0-677-gf7870454-1jammy [30.7 kB] 2026-02-20T21:36:39.349 INFO:teuthology.orchestra.run.trial107.stdout:Get:84 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 20.2.0-677-gf7870454-1jammy [41.5 kB] 2026-02-20T21:36:39.349 INFO:teuthology.orchestra.run.trial107.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 20.2.0-677-gf7870454-1jammy [25.1 MB] 2026-02-20T21:36:39.920 INFO:teuthology.orchestra.run.trial107.stdout:Get:86 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 20.2.0-677-gf7870454-1jammy [97.9 kB] 2026-02-20T21:36:39.920 INFO:teuthology.orchestra.run.trial107.stdout:Get:87 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 20.2.0-677-gf7870454-1jammy [2883 kB] 2026-02-20T21:36:40.264 INFO:teuthology.orchestra.run.trial107.stdout:Fetched 283 MB in 6s (45.1 MB/s) 2026-02-20T21:36:40.281 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-20T21:36:40.365 INFO:teuthology.orchestra.run.trial032.stdout:Get:78 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 20.2.0-677-gf7870454-1jammy [8625 kB] 2026-02-20T21:36:40.440 INFO:teuthology.orchestra.run.trial107.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167442 files and directories currently installed.) 2026-02-20T21:36:40.442 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../00-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-20T21:36:40.443 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-20T21:36:40.464 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-20T21:36:40.475 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../01-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-20T21:36:40.475 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-20T21:36:40.492 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-20T21:36:40.500 INFO:teuthology.orchestra.run.trial032.stdout:Get:79 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 20.2.0-677-gf7870454-1jammy [14.2 kB] 2026-02-20T21:36:40.500 INFO:teuthology.orchestra.run.trial032.stdout:Get:80 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 20.2.0-677-gf7870454-1jammy [99.5 MB] 2026-02-20T21:36:40.504 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../02-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-20T21:36:40.505 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-20T21:36:40.524 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-20T21:36:40.535 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../03-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-20T21:36:40.540 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:40.572 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-20T21:36:40.580 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../04-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-20T21:36:40.580 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:40.596 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-20T21:36:40.604 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../05-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-20T21:36:40.605 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:40.624 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-20T21:36:40.632 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../06-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-20T21:36:40.633 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-20T21:36:40.648 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package librados2. 2026-02-20T21:36:40.656 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../07-librados2_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:40.656 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking librados2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:40.698 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libnbd0. 2026-02-20T21:36:40.706 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../08-libnbd0_1.10.5-1_amd64.deb ... 2026-02-20T21:36:40.707 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-20T21:36:40.721 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package librbd1. 2026-02-20T21:36:40.729 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../09-librbd1_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:40.729 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking librbd1 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:40.767 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libcephfs2. 2026-02-20T21:36:40.775 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../10-libcephfs2_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:40.776 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libcephfs2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:40.795 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-rados. 2026-02-20T21:36:40.804 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../11-python3-rados_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:40.804 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-rados (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:40.821 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-20T21:36:40.829 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../12-python3-ceph-argparse_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:40.829 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-ceph-argparse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:40.843 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-cephfs. 2026-02-20T21:36:40.851 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../13-python3-cephfs_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:40.852 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-cephfs (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:40.867 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-20T21:36:40.876 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../14-python3-ceph-common_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:40.876 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-ceph-common (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:40.893 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-20T21:36:40.902 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../15-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-20T21:36:40.902 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-20T21:36:40.917 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-prettytable. 2026-02-20T21:36:40.925 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../16-python3-prettytable_2.5.0-2_all.deb ... 2026-02-20T21:36:40.926 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-20T21:36:40.940 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-rbd. 2026-02-20T21:36:40.948 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../17-python3-rbd_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:40.948 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-rbd (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:40.965 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-20T21:36:40.973 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../18-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-20T21:36:40.973 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-20T21:36:40.992 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package librgw2. 2026-02-20T21:36:41.000 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../19-librgw2_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:41.000 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking librgw2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:41.113 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-rgw. 2026-02-20T21:36:41.121 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../20-python3-rgw_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:41.122 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-rgw (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:41.137 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-20T21:36:41.145 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../21-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-20T21:36:41.145 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-20T21:36:41.160 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libradosstriper1. 2026-02-20T21:36:41.168 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../22-libradosstriper1_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:41.168 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libradosstriper1 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:41.185 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-common. 2026-02-20T21:36:41.193 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../23-ceph-common_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:41.193 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-common (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:41.511 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-base. 2026-02-20T21:36:41.519 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../24-ceph-base_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:41.524 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-base (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:41.590 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-20T21:36:41.599 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../25-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-20T21:36:41.599 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-20T21:36:41.613 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-cheroot. 2026-02-20T21:36:41.621 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../26-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-20T21:36:41.622 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-20T21:36:41.638 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-20T21:36:41.646 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../27-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-20T21:36:41.646 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-20T21:36:41.660 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-20T21:36:41.668 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../28-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-20T21:36:41.669 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-20T21:36:41.684 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-20T21:36:41.692 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../29-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-20T21:36:41.692 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-20T21:36:41.706 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-tempora. 2026-02-20T21:36:41.714 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../30-python3-tempora_4.1.2-1_all.deb ... 2026-02-20T21:36:41.714 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-20T21:36:41.729 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-portend. 2026-02-20T21:36:41.737 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../31-python3-portend_3.0.0-1_all.deb ... 2026-02-20T21:36:41.737 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-20T21:36:41.751 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-20T21:36:41.759 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../32-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-20T21:36:41.759 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-20T21:36:41.774 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-20T21:36:41.782 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../33-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-20T21:36:41.783 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-20T21:36:41.806 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-natsort. 2026-02-20T21:36:41.814 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../34-python3-natsort_8.0.2-1_all.deb ... 2026-02-20T21:36:41.814 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-20T21:36:41.830 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-20T21:36:41.838 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../35-ceph-mgr-modules-core_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:41.839 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-mgr-modules-core (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:41.865 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-20T21:36:41.873 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../36-libsqlite3-mod-ceph_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:41.873 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libsqlite3-mod-ceph (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:41.889 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-mgr. 2026-02-20T21:36:41.897 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../37-ceph-mgr_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:41.897 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-mgr (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:41.918 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-mon. 2026-02-20T21:36:41.926 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../38-ceph-mon_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:41.927 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-mon (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:41.983 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-20T21:36:41.991 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../39-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-20T21:36:41.992 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-20T21:36:42.007 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-osd. 2026-02-20T21:36:42.016 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../40-ceph-osd_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:42.016 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-osd (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:42.145 INFO:teuthology.orchestra.run.trial032.stdout:Get:81 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 20.2.0-677-gf7870454-1jammy [135 kB] 2026-02-20T21:36:42.146 INFO:teuthology.orchestra.run.trial032.stdout:Get:82 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-daemon amd64 20.2.0-677-gf7870454-1jammy [43.3 kB] 2026-02-20T21:36:42.146 INFO:teuthology.orchestra.run.trial032.stdout:Get:83 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-proxy2 amd64 20.2.0-677-gf7870454-1jammy [30.7 kB] 2026-02-20T21:36:42.146 INFO:teuthology.orchestra.run.trial032.stdout:Get:84 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 20.2.0-677-gf7870454-1jammy [41.5 kB] 2026-02-20T21:36:42.147 INFO:teuthology.orchestra.run.trial032.stdout:Get:85 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 20.2.0-677-gf7870454-1jammy [25.1 MB] 2026-02-20T21:36:42.164 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph. 2026-02-20T21:36:42.172 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../41-ceph_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:42.173 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:42.186 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-fuse. 2026-02-20T21:36:42.194 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../42-ceph-fuse_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:42.195 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-fuse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:42.216 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-mds. 2026-02-20T21:36:42.225 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../43-ceph-mds_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:42.225 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-mds (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:42.257 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package cephadm. 2026-02-20T21:36:42.265 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../44-cephadm_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:42.266 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking cephadm (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:42.282 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-20T21:36:42.290 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../45-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-20T21:36:42.291 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-20T21:36:42.311 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-20T21:36:42.320 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../46-ceph-mgr-cephadm_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:42.320 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-mgr-cephadm (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:42.341 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-20T21:36:42.349 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../47-python3-repoze.lru_0.7-2_all.deb ... 2026-02-20T21:36:42.349 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-20T21:36:42.364 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-routes. 2026-02-20T21:36:42.372 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../48-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-20T21:36:42.373 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-20T21:36:42.392 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-20T21:36:42.400 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../49-ceph-mgr-dashboard_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:42.401 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-mgr-dashboard (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:42.550 INFO:teuthology.orchestra.run.trial032.stdout:Get:86 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 20.2.0-677-gf7870454-1jammy [97.9 kB] 2026-02-20T21:36:42.550 INFO:teuthology.orchestra.run.trial032.stdout:Get:87 https://1.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 20.2.0-677-gf7870454-1jammy [2883 kB] 2026-02-20T21:36:42.834 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-20T21:36:42.844 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../50-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-20T21:36:42.845 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-20T21:36:42.871 INFO:teuthology.orchestra.run.trial032.stdout:Fetched 283 MB in 9s (31.9 MB/s) 2026-02-20T21:36:42.883 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-joblib. 2026-02-20T21:36:42.891 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-20T21:36:42.892 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../51-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-20T21:36:42.892 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-20T21:36:42.918 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-20T21:36:42.926 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../52-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-20T21:36:42.927 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-20T21:36:42.942 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-sklearn. 2026-02-20T21:36:42.951 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../53-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-20T21:36:42.951 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-20T21:36:43.034 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-20T21:36:43.042 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../54-ceph-mgr-diskprediction-local_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:43.043 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-mgr-diskprediction-local (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.050 INFO:teuthology.orchestra.run.trial032.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167442 files and directories currently installed.) 2026-02-20T21:36:43.052 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../00-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-20T21:36:43.054 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-20T21:36:43.076 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-20T21:36:43.089 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../01-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-20T21:36:43.089 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-20T21:36:43.105 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-20T21:36:43.117 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../02-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-20T21:36:43.117 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-20T21:36:43.139 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-20T21:36:43.150 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../03-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-20T21:36:43.155 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:43.190 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-20T21:36:43.199 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../04-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-20T21:36:43.200 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:43.204 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-cachetools. 2026-02-20T21:36:43.213 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../55-python3-cachetools_5.0.0-1_all.deb ... 2026-02-20T21:36:43.213 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-20T21:36:43.215 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-20T21:36:43.223 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../05-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-20T21:36:43.224 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:43.228 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-rsa. 2026-02-20T21:36:43.236 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../56-python3-rsa_4.8-1_all.deb ... 2026-02-20T21:36:43.236 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-20T21:36:43.245 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-20T21:36:43.253 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../06-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-20T21:36:43.253 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-20T21:36:43.255 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-google-auth. 2026-02-20T21:36:43.264 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../57-python3-google-auth_1.5.1-3_all.deb ... 2026-02-20T21:36:43.264 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-20T21:36:43.271 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package librados2. 2026-02-20T21:36:43.278 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../07-librados2_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.279 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking librados2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.281 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-20T21:36:43.290 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../58-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-20T21:36:43.290 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-20T21:36:43.305 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-websocket. 2026-02-20T21:36:43.314 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../59-python3-websocket_1.2.3-1_all.deb ... 2026-02-20T21:36:43.314 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-20T21:36:43.320 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libnbd0. 2026-02-20T21:36:43.328 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../08-libnbd0_1.10.5-1_amd64.deb ... 2026-02-20T21:36:43.329 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-20T21:36:43.330 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-20T21:36:43.338 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../60-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-20T21:36:43.339 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-20T21:36:43.342 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package librbd1. 2026-02-20T21:36:43.351 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../09-librbd1_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.351 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking librbd1 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.387 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libcephfs2. 2026-02-20T21:36:43.395 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../10-libcephfs2_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.396 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libcephfs2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.415 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-rados. 2026-02-20T21:36:43.423 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../11-python3-rados_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.423 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-rados (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.430 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-20T21:36:43.439 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../61-ceph-mgr-k8sevents_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:43.439 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-mgr-k8sevents (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.441 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-20T21:36:43.448 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../12-python3-ceph-argparse_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:43.449 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-ceph-argparse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.453 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-20T21:36:43.462 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../62-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-20T21:36:43.462 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-20T21:36:43.462 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-cephfs. 2026-02-20T21:36:43.471 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../13-python3-cephfs_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.471 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-cephfs (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.478 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-20T21:36:43.487 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-20T21:36:43.487 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../63-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-20T21:36:43.488 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-20T21:36:43.494 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../14-python3-ceph-common_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:43.495 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-ceph-common (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.502 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package jq. 2026-02-20T21:36:43.510 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../64-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-20T21:36:43.510 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-20T21:36:43.513 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-20T21:36:43.521 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../15-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-20T21:36:43.522 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-20T21:36:43.524 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package socat. 2026-02-20T21:36:43.533 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../65-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-20T21:36:43.533 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-20T21:36:43.537 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-prettytable. 2026-02-20T21:36:43.545 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../16-python3-prettytable_2.5.0-2_all.deb ... 2026-02-20T21:36:43.546 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-20T21:36:43.553 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package xmlstarlet. 2026-02-20T21:36:43.560 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-rbd. 2026-02-20T21:36:43.561 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../66-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-20T21:36:43.561 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-20T21:36:43.568 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../17-python3-rbd_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.568 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-rbd (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.586 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-20T21:36:43.593 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../18-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-20T21:36:43.594 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-20T21:36:43.596 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-test. 2026-02-20T21:36:43.604 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../67-ceph-test_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.605 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-test (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.612 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package librgw2. 2026-02-20T21:36:43.620 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../19-librgw2_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.620 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking librgw2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.737 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-rgw. 2026-02-20T21:36:43.745 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../20-python3-rgw_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.746 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-rgw (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.761 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-20T21:36:43.768 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../21-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-20T21:36:43.769 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-20T21:36:43.783 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libradosstriper1. 2026-02-20T21:36:43.791 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../22-libradosstriper1_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.792 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libradosstriper1 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:43.807 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-common. 2026-02-20T21:36:43.815 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../23-ceph-common_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:43.815 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-common (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.062 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-base. 2026-02-20T21:36:44.071 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../24-ceph-base_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.075 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-base (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.141 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-20T21:36:44.149 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../25-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-20T21:36:44.149 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-20T21:36:44.164 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-cheroot. 2026-02-20T21:36:44.172 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../26-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-20T21:36:44.172 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-20T21:36:44.189 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-20T21:36:44.197 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../27-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-20T21:36:44.198 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-20T21:36:44.212 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-20T21:36:44.221 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../28-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-20T21:36:44.221 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-20T21:36:44.235 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-20T21:36:44.243 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../29-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-20T21:36:44.244 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-20T21:36:44.257 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-tempora. 2026-02-20T21:36:44.266 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../30-python3-tempora_4.1.2-1_all.deb ... 2026-02-20T21:36:44.266 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-20T21:36:44.281 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-portend. 2026-02-20T21:36:44.289 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../31-python3-portend_3.0.0-1_all.deb ... 2026-02-20T21:36:44.290 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-20T21:36:44.303 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-20T21:36:44.311 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../32-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-20T21:36:44.311 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-20T21:36:44.326 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-20T21:36:44.334 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../33-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-20T21:36:44.334 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-20T21:36:44.357 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-natsort. 2026-02-20T21:36:44.365 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../34-python3-natsort_8.0.2-1_all.deb ... 2026-02-20T21:36:44.366 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-20T21:36:44.381 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-20T21:36:44.389 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../35-ceph-mgr-modules-core_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:44.390 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-mgr-modules-core (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.416 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-20T21:36:44.424 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../36-libsqlite3-mod-ceph_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.424 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libsqlite3-mod-ceph (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.439 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-mgr. 2026-02-20T21:36:44.448 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../37-ceph-mgr_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.448 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-mgr (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.469 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-mon. 2026-02-20T21:36:44.477 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../38-ceph-mon_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.478 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-mon (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.535 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-20T21:36:44.537 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package ceph-volume. 2026-02-20T21:36:44.543 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../39-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-20T21:36:44.544 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-20T21:36:44.546 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../68-ceph-volume_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:44.546 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking ceph-volume (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.560 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-osd. 2026-02-20T21:36:44.568 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../40-ceph-osd_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.568 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-osd (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.569 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libcephfs-daemon. 2026-02-20T21:36:44.577 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../69-libcephfs-daemon_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.577 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libcephfs-daemon (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.591 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libcephfs-proxy2. 2026-02-20T21:36:44.600 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../70-libcephfs-proxy2_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.600 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libcephfs-proxy2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.614 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-20T21:36:44.623 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../71-libcephfs-dev_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.623 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking libcephfs-dev (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.638 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package nvme-cli. 2026-02-20T21:36:44.647 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../72-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-20T21:36:44.647 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-20T21:36:44.679 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-20T21:36:44.687 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../73-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-20T21:36:44.687 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-20T21:36:44.715 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph. 2026-02-20T21:36:44.717 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-20T21:36:44.724 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../41-ceph_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.724 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.725 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../74-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-20T21:36:44.725 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-20T21:36:44.739 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-fuse. 2026-02-20T21:36:44.740 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-pluggy. 2026-02-20T21:36:44.747 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../42-ceph-fuse_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.747 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-fuse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.749 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../75-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-20T21:36:44.749 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-20T21:36:44.765 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-psutil. 2026-02-20T21:36:44.768 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-mds. 2026-02-20T21:36:44.773 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../76-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-20T21:36:44.774 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-20T21:36:44.777 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../43-ceph-mds_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.777 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-mds (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.791 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-py. 2026-02-20T21:36:44.800 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../77-python3-py_1.10.0-1_all.deb ... 2026-02-20T21:36:44.800 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-20T21:36:44.809 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package cephadm. 2026-02-20T21:36:44.817 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../44-cephadm_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:44.817 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking cephadm (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.819 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-pygments. 2026-02-20T21:36:44.828 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../78-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-20T21:36:44.828 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-20T21:36:44.833 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-20T21:36:44.841 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../45-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-20T21:36:44.842 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-20T21:36:44.863 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-20T21:36:44.870 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-toml. 2026-02-20T21:36:44.871 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../46-ceph-mgr-cephadm_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:44.871 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-mgr-cephadm (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.879 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../79-python3-toml_0.10.2-1_all.deb ... 2026-02-20T21:36:44.880 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-20T21:36:44.892 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-20T21:36:44.895 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-pytest. 2026-02-20T21:36:44.900 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../47-python3-repoze.lru_0.7-2_all.deb ... 2026-02-20T21:36:44.901 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-20T21:36:44.903 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../80-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-20T21:36:44.903 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-20T21:36:44.916 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-routes. 2026-02-20T21:36:44.923 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../48-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-20T21:36:44.924 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-20T21:36:44.925 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-simplejson. 2026-02-20T21:36:44.934 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../81-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-20T21:36:44.934 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-20T21:36:44.943 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-20T21:36:44.951 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../49-ceph-mgr-dashboard_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:44.952 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package python3-webob. 2026-02-20T21:36:44.952 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-mgr-dashboard (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:44.960 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../82-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-20T21:36:44.960 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-20T21:36:44.984 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-20T21:36:44.992 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../83-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-20T21:36:44.993 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-20T21:36:45.061 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package radosgw. 2026-02-20T21:36:45.070 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../84-radosgw_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:45.070 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking radosgw (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:45.296 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package rbd-fuse. 2026-02-20T21:36:45.305 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../85-rbd-fuse_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:45.305 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking rbd-fuse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:45.321 INFO:teuthology.orchestra.run.trial107.stdout:Selecting previously unselected package rbd-mirror. 2026-02-20T21:36:45.329 INFO:teuthology.orchestra.run.trial107.stdout:Preparing to unpack .../86-rbd-mirror_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:45.329 INFO:teuthology.orchestra.run.trial107.stdout:Unpacking rbd-mirror (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:45.377 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-20T21:36:45.384 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-20T21:36:45.386 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../50-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-20T21:36:45.386 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-20T21:36:45.425 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-joblib. 2026-02-20T21:36:45.433 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../51-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-20T21:36:45.433 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-20T21:36:45.449 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-20T21:36:45.449 INFO:teuthology.orchestra.run.trial107.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-20T21:36:45.460 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-20T21:36:45.469 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../52-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-20T21:36:45.469 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-20T21:36:45.484 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-sklearn. 2026-02-20T21:36:45.492 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../53-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-20T21:36:45.493 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-20T21:36:45.521 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-20T21:36:45.575 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-20T21:36:45.584 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../54-ceph-mgr-diskprediction-local_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:45.584 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-mgr-diskprediction-local (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:45.748 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-cachetools. 2026-02-20T21:36:45.756 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../55-python3-cachetools_5.0.0-1_all.deb ... 2026-02-20T21:36:45.757 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-20T21:36:45.771 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-rsa. 2026-02-20T21:36:45.780 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../56-python3-rsa_4.8-1_all.deb ... 2026-02-20T21:36:45.780 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-20T21:36:45.786 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-20T21:36:45.798 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-google-auth. 2026-02-20T21:36:45.807 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../57-python3-google-auth_1.5.1-3_all.deb ... 2026-02-20T21:36:45.807 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-20T21:36:45.825 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-20T21:36:45.834 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../58-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-20T21:36:45.834 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-20T21:36:45.850 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-websocket. 2026-02-20T21:36:45.858 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../59-python3-websocket_1.2.3-1_all.deb ... 2026-02-20T21:36:45.859 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-20T21:36:45.875 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-20T21:36:45.884 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../60-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-20T21:36:45.884 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-20T21:36:45.977 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-20T21:36:45.986 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../61-ceph-mgr-k8sevents_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:45.987 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-mgr-k8sevents (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:46.000 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-20T21:36:46.009 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../62-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-20T21:36:46.009 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-20T21:36:46.025 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-20T21:36:46.033 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../63-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-20T21:36:46.034 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-20T21:36:46.048 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package jq. 2026-02-20T21:36:46.057 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../64-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-20T21:36:46.057 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-20T21:36:46.071 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package socat. 2026-02-20T21:36:46.080 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../65-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-20T21:36:46.080 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-20T21:36:46.100 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package xmlstarlet. 2026-02-20T21:36:46.109 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../66-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-20T21:36:46.109 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-20T21:36:46.145 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-test. 2026-02-20T21:36:46.153 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../67-ceph-test_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:46.154 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-test (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:46.168 INFO:teuthology.orchestra.run.trial107.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-20T21:36:46.178 INFO:teuthology.orchestra.run.trial107.stdout:Setting up cephadm (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:46.220 INFO:teuthology.orchestra.run.trial107.stdout:Adding system user cephadm....done 2026-02-20T21:36:46.230 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-20T21:36:46.292 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-20T21:36:46.293 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-20T21:36:46.364 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-20T21:36:46.439 INFO:teuthology.orchestra.run.trial107.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-20T21:36:46.440 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-20T21:36:46.535 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-20T21:36:46.658 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-20T21:36:46.735 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-20T21:36:46.809 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-ceph-argparse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:46.875 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-20T21:36:46.876 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-20T21:36:46.877 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-20T21:36:46.878 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-20T21:36:46.879 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-20T21:36:46.998 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-20T21:36:47.044 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package ceph-volume. 2026-02-20T21:36:47.052 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../68-ceph-volume_20.2.0-677-gf7870454-1jammy_all.deb ... 2026-02-20T21:36:47.053 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking ceph-volume (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:47.076 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libcephfs-daemon. 2026-02-20T21:36:47.077 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libcephfs-proxy2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:47.078 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-20T21:36:47.085 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../69-libcephfs-daemon_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:47.085 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libcephfs-daemon (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:47.101 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libcephfs-proxy2. 2026-02-20T21:36:47.109 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../70-libcephfs-proxy2_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:47.110 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libcephfs-proxy2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:47.124 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-20T21:36:47.133 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../71-libcephfs-dev_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:47.133 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking libcephfs-dev (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:47.150 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package nvme-cli. 2026-02-20T21:36:47.154 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-20T21:36:47.158 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../72-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-20T21:36:47.158 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-20T21:36:47.190 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-20T21:36:47.199 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../73-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-20T21:36:47.199 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-20T21:36:47.228 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-20T21:36:47.237 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../74-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-20T21:36:47.237 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-20T21:36:47.246 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-20T21:36:47.252 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-pluggy. 2026-02-20T21:36:47.260 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../75-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-20T21:36:47.261 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-20T21:36:47.276 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-psutil. 2026-02-20T21:36:47.285 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../76-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-20T21:36:47.285 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-20T21:36:47.305 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-py. 2026-02-20T21:36:47.313 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../77-python3-py_1.10.0-1_all.deb ... 2026-02-20T21:36:47.313 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-20T21:36:47.333 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-pygments. 2026-02-20T21:36:47.342 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../78-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-20T21:36:47.342 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-20T21:36:47.385 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-toml. 2026-02-20T21:36:47.393 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../79-python3-toml_0.10.2-1_all.deb ... 2026-02-20T21:36:47.394 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-20T21:36:47.409 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-pytest. 2026-02-20T21:36:47.417 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../80-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-20T21:36:47.417 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-20T21:36:47.439 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-simplejson. 2026-02-20T21:36:47.448 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../81-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-20T21:36:47.448 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-20T21:36:47.466 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package python3-webob. 2026-02-20T21:36:47.474 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../82-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-20T21:36:47.474 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-20T21:36:47.497 INFO:teuthology.orchestra.run.trial107.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-20T21:36:47.498 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-20T21:36:47.498 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-20T21:36:47.506 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../83-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-20T21:36:47.507 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-20T21:36:47.573 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package radosgw. 2026-02-20T21:36:47.581 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../84-radosgw_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:47.582 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking radosgw (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:47.594 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-20T21:36:47.723 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-20T21:36:47.805 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-20T21:36:47.847 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package rbd-fuse. 2026-02-20T21:36:47.855 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../85-rbd-fuse_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:47.856 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking rbd-fuse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:47.871 INFO:teuthology.orchestra.run.trial032.stdout:Selecting previously unselected package rbd-mirror. 2026-02-20T21:36:47.876 INFO:teuthology.orchestra.run.trial107.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-20T21:36:47.877 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-ceph-common (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:47.880 INFO:teuthology.orchestra.run.trial032.stdout:Preparing to unpack .../86-rbd-mirror_20.2.0-677-gf7870454-1jammy_amd64.deb ... 2026-02-20T21:36:47.880 INFO:teuthology.orchestra.run.trial032.stdout:Unpacking rbd-mirror (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:47.934 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-20T21:36:47.964 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-20T21:36:48.000 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-20T21:36:48.001 INFO:teuthology.orchestra.run.trial032.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-20T21:36:48.072 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-20T21:36:48.349 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-20T21:36:48.464 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:48.469 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-20T21:36:48.545 INFO:teuthology.orchestra.run.trial107.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-20T21:36:48.546 INFO:teuthology.orchestra.run.trial107.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-20T21:36:48.547 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-20T21:36:48.620 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-20T21:36:48.692 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:48.693 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-20T21:36:48.740 INFO:teuthology.orchestra.run.trial032.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-20T21:36:48.751 INFO:teuthology.orchestra.run.trial032.stdout:Setting up cephadm (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:48.770 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-20T21:36:48.793 INFO:teuthology.orchestra.run.trial032.stdout:Adding system user cephadm....done 2026-02-20T21:36:48.804 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-20T21:36:48.847 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-20T21:36:48.880 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-20T21:36:48.881 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-20T21:36:48.925 INFO:teuthology.orchestra.run.trial107.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-20T21:36:48.926 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-20T21:36:48.954 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-20T21:36:49.011 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-20T21:36:49.012 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-20T21:36:49.028 INFO:teuthology.orchestra.run.trial032.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-20T21:36:49.029 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-20T21:36:49.088 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-20T21:36:49.124 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-20T21:36:49.173 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-20T21:36:49.246 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-20T21:36:49.246 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-20T21:36:49.247 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-20T21:36:49.320 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-20T21:36:49.382 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-20T21:36:49.395 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-ceph-argparse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:49.459 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:49.460 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-20T21:36:49.464 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-20T21:36:49.465 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-20T21:36:49.466 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-20T21:36:49.467 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-20T21:36:49.468 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-20T21:36:49.542 INFO:teuthology.orchestra.run.trial107.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-20T21:36:49.543 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-20T21:36:49.588 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-20T21:36:49.663 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libcephfs-proxy2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:49.663 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-20T21:36:49.674 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-20T21:36:49.675 INFO:teuthology.orchestra.run.trial107.stdout:Setting up librados2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:49.676 INFO:teuthology.orchestra.run.trial107.stdout:Setting up librgw2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:49.677 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libsqlite3-mod-ceph (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:49.678 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-20T21:36:49.743 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-20T21:36:49.829 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-20T21:36:50.083 INFO:teuthology.orchestra.run.trial032.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-20T21:36:50.084 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-20T21:36:50.180 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-20T21:36:50.204 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libcephfs2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.205 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libradosstriper1 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.206 INFO:teuthology.orchestra.run.trial107.stdout:Setting up librbd1 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.207 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-mgr-modules-core (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.208 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-fuse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.281 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-20T21:36:50.281 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-20T21:36:50.305 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-20T21:36:50.386 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-20T21:36:50.458 INFO:teuthology.orchestra.run.trial032.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-20T21:36:50.459 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-ceph-common (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.546 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-20T21:36:50.657 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libcephfs-dev (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.658 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-rados (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.659 INFO:teuthology.orchestra.run.trial107.stdout:Setting up libcephfs-daemon (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.660 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-rbd (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.661 INFO:teuthology.orchestra.run.trial107.stdout:Setting up rbd-fuse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.662 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-rgw (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.663 INFO:teuthology.orchestra.run.trial107.stdout:Setting up python3-cephfs (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.664 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-common (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:50.693 INFO:teuthology.orchestra.run.trial107.stdout:Adding group ceph....done 2026-02-20T21:36:50.727 INFO:teuthology.orchestra.run.trial107.stdout:Adding system user ceph....done 2026-02-20T21:36:50.736 INFO:teuthology.orchestra.run.trial107.stdout:Setting system user ceph properties....done 2026-02-20T21:36:50.742 INFO:teuthology.orchestra.run.trial107.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-20T21:36:50.830 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-20T21:36:51.049 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:51.054 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-20T21:36:51.091 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-20T21:36:51.129 INFO:teuthology.orchestra.run.trial032.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-20T21:36:51.130 INFO:teuthology.orchestra.run.trial032.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-20T21:36:51.131 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-20T21:36:51.207 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-20T21:36:51.281 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:51.282 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-20T21:36:51.362 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-20T21:36:51.435 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-20T21:36:51.461 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-test (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:51.462 INFO:teuthology.orchestra.run.trial107.stdout:Setting up radosgw (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:51.511 INFO:teuthology.orchestra.run.trial032.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-20T21:36:51.512 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-20T21:36:51.596 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-20T21:36:51.597 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-20T21:36:51.672 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-20T21:36:51.708 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:36:51.708 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:36:51.756 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-20T21:36:51.830 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-20T21:36:51.831 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-20T21:36:51.967 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-20T21:36:52.040 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-20T21:36:52.041 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-20T21:36:52.077 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-base (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.126 INFO:teuthology.orchestra.run.trial032.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-20T21:36:52.127 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-20T21:36:52.195 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-20T21:36:52.256 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-20T21:36:52.257 INFO:teuthology.orchestra.run.trial032.stdout:Setting up librados2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.258 INFO:teuthology.orchestra.run.trial032.stdout:Setting up librgw2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.259 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libsqlite3-mod-ceph (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.260 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-20T21:36:52.561 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-mds (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.620 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-20T21:36:52.620 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-20T21:36:52.778 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libcephfs2 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.779 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libradosstriper1 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.780 INFO:teuthology.orchestra.run.trial032.stdout:Setting up librbd1 (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.781 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-mgr-modules-core (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.782 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-fuse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:52.852 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-20T21:36:52.853 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-20T21:36:52.962 INFO:teuthology.orchestra.run.trial107.stdout:Setting up rbd-mirror (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.020 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:36:53.020 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:36:53.259 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libcephfs-dev (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.260 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-rados (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.261 INFO:teuthology.orchestra.run.trial032.stdout:Setting up libcephfs-daemon (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.261 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-rbd (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.262 INFO:teuthology.orchestra.run.trial032.stdout:Setting up rbd-fuse (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.263 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-rgw (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.264 INFO:teuthology.orchestra.run.trial032.stdout:Setting up python3-cephfs (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.265 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-common (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.293 INFO:teuthology.orchestra.run.trial032.stdout:Adding group ceph....done 2026-02-20T21:36:53.327 INFO:teuthology.orchestra.run.trial032.stdout:Adding system user ceph....done 2026-02-20T21:36:53.336 INFO:teuthology.orchestra.run.trial032.stdout:Setting system user ceph properties....done 2026-02-20T21:36:53.342 INFO:teuthology.orchestra.run.trial032.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-20T21:36:53.369 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-mgr (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.429 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-20T21:36:53.444 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-20T21:36:53.444 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-20T21:36:53.720 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-20T21:36:53.821 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-osd (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:53.903 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-20T21:36:53.904 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-20T21:36:54.111 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-test (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.112 INFO:teuthology.orchestra.run.trial032.stdout:Setting up radosgw (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.262 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-mgr-k8sevents (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.263 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-mgr-diskprediction-local (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.273 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-mon (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.345 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-20T21:36:54.345 INFO:teuthology.orchestra.run.trial107.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-20T21:36:54.381 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:36:54.381 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:36:54.731 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-mgr-cephadm (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.742 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.743 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-mgr-dashboard (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.753 INFO:teuthology.orchestra.run.trial107.stdout:Setting up ceph-volume (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.757 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-base (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:54.864 INFO:teuthology.orchestra.run.trial107.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-20T21:36:54.868 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-20T21:36:54.885 INFO:teuthology.orchestra.run.trial107.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-02-20T21:36:55.276 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-mds (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:55.352 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-20T21:36:55.352 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-20T21:36:55.538 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:36:55.539 INFO:teuthology.orchestra.run.trial107.stdout:Running kernel seems to be up-to-date. 2026-02-20T21:36:55.539 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:36:55.539 INFO:teuthology.orchestra.run.trial107.stdout:The processor microcode seems to be up-to-date. 2026-02-20T21:36:55.539 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:36:55.539 INFO:teuthology.orchestra.run.trial107.stdout:No services need to be restarted. 2026-02-20T21:36:55.539 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:36:55.539 INFO:teuthology.orchestra.run.trial107.stdout:No containers need to be restarted. 2026-02-20T21:36:55.539 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:36:55.539 INFO:teuthology.orchestra.run.trial107.stdout:No user sessions are running outdated binaries. 2026-02-20T21:36:55.540 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:36:55.540 INFO:teuthology.orchestra.run.trial107.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-20T21:36:55.742 INFO:teuthology.orchestra.run.trial032.stdout:Setting up rbd-mirror (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:55.817 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:36:55.817 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:36:56.197 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-mgr (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:56.277 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-20T21:36:56.277 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-20T21:36:56.352 INFO:teuthology.orchestra.run.trial107.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-20T21:36:56.361 DEBUG:teuthology.parallel:result is None 2026-02-20T21:36:56.658 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-osd (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:56.764 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-20T21:36:56.764 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-20T21:36:57.175 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-mgr-k8sevents (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:57.176 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-mgr-diskprediction-local (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:57.186 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-mon (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:57.259 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-20T21:36:57.259 INFO:teuthology.orchestra.run.trial032.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-20T21:36:57.694 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-mgr-cephadm (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:57.705 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:57.706 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-mgr-dashboard (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:57.716 INFO:teuthology.orchestra.run.trial032.stdout:Setting up ceph-volume (20.2.0-677-gf7870454-1jammy) ... 2026-02-20T21:36:57.826 INFO:teuthology.orchestra.run.trial032.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-20T21:36:57.911 INFO:teuthology.orchestra.run.trial032.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-02-20T21:36:58.575 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:36:58.575 INFO:teuthology.orchestra.run.trial032.stdout:Running kernel seems to be up-to-date. 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout:The processor microcode seems to be up-to-date. 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout:No services need to be restarted. 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout:No containers need to be restarted. 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout:No user sessions are running outdated binaries. 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:36:58.576 INFO:teuthology.orchestra.run.trial032.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-20T21:36:59.385 INFO:teuthology.orchestra.run.trial032.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-20T21:36:59.392 DEBUG:teuthology.parallel:result is None 2026-02-20T21:36:59.393 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:36:59.549 DEBUG:teuthology.orchestra.run.trial032:> dpkg-query -W -f '${Version}' ceph 2026-02-20T21:36:59.560 INFO:teuthology.orchestra.run.trial032.stdout:20.2.0-677-gf7870454-1jammy 2026-02-20T21:36:59.560 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-677-gf7870454-1jammy 2026-02-20T21:36:59.560 INFO:teuthology.task.install:The correct ceph version 20.2.0-677-gf7870454-1jammy is installed. 2026-02-20T21:36:59.562 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:36:59.686 DEBUG:teuthology.orchestra.run.trial107:> dpkg-query -W -f '${Version}' ceph 2026-02-20T21:36:59.697 INFO:teuthology.orchestra.run.trial107.stdout:20.2.0-677-gf7870454-1jammy 2026-02-20T21:36:59.697 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-677-gf7870454-1jammy 2026-02-20T21:36:59.697 INFO:teuthology.task.install:The correct ceph version 20.2.0-677-gf7870454-1jammy is installed. 2026-02-20T21:36:59.699 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-20T21:36:59.699 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:36:59.699 DEBUG:teuthology.orchestra.run.trial032:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-20T21:36:59.710 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:36:59.710 DEBUG:teuthology.orchestra.run.trial107:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-20T21:36:59.747 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-20T21:36:59.748 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:36:59.748 DEBUG:teuthology.orchestra.run.trial032:> sudo dd of=/usr/bin/daemon-helper 2026-02-20T21:36:59.761 DEBUG:teuthology.orchestra.run.trial032:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-20T21:36:59.812 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:36:59.812 DEBUG:teuthology.orchestra.run.trial107:> sudo dd of=/usr/bin/daemon-helper 2026-02-20T21:36:59.820 DEBUG:teuthology.orchestra.run.trial107:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-20T21:36:59.872 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-20T21:36:59.872 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:36:59.872 DEBUG:teuthology.orchestra.run.trial032:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-20T21:36:59.880 DEBUG:teuthology.orchestra.run.trial032:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-20T21:36:59.931 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:36:59.932 DEBUG:teuthology.orchestra.run.trial107:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-20T21:36:59.940 DEBUG:teuthology.orchestra.run.trial107:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-20T21:36:59.990 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-20T21:36:59.991 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:36:59.991 DEBUG:teuthology.orchestra.run.trial032:> sudo dd of=/usr/bin/stdin-killer 2026-02-20T21:37:00.001 DEBUG:teuthology.orchestra.run.trial032:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-20T21:37:00.051 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:00.051 DEBUG:teuthology.orchestra.run.trial107:> sudo dd of=/usr/bin/stdin-killer 2026-02-20T21:37:00.060 DEBUG:teuthology.orchestra.run.trial107:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-20T21:37:00.111 INFO:teuthology.run_tasks:Running task ceph... 2026-02-20T21:37:00.218 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-20T21:37:00.218 DEBUG:teuthology.orchestra.run.trial032:> sudo chmod 777 /var/log/ceph 2026-02-20T21:37:00.220 DEBUG:teuthology.orchestra.run.trial107:> sudo chmod 777 /var/log/ceph 2026-02-20T21:37:00.228 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-20T21:37:00.228 DEBUG:teuthology.orchestra.run.trial032:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-20T21:37:00.268 DEBUG:teuthology.orchestra.run.trial107:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-20T21:37:00.278 INFO:tasks.ceph:Creating extra log directories... 2026-02-20T21:37:00.278 DEBUG:teuthology.orchestra.run.trial032:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-20T21:37:00.321 DEBUG:teuthology.orchestra.run.trial107:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-20T21:37:00.349 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-20T21:37:00.350 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-20T21:37:00.350 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486', 'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-snappy policy/none rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-stress-workunit-fast-diff}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '61486', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial', 'no_nested_subset': False, 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7161, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f7870454149f22574086dd2ff55d050585136531', 'targets': {'trial032.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPRXC6A0nz0Ujzi5yHHzygV4NFoxUEbgNW5j0qSKAWKxFkkbu/brPWlpCdqn6tyLIGv6fhrzujk2pJ6qXpMi07s=', 'trial107.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG0Ox7QcjWoqAVtqdwVfCJ6pcSiRRJmVoXtIhl0//qNUjkwWwsjgx3aj2wx7whWZfyuqAMUwwncaoPTMGgKoBzo='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock,object-map,fast-diff', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-20_16:15:01', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699'} 2026-02-20T21:37:00.351 DEBUG:teuthology.orchestra.run.trial032:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-20T21:37:00.373 DEBUG:teuthology.orchestra.run.trial107:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-20T21:37:00.378 DEBUG:teuthology.orchestra.run.trial032:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:37:00.417 DEBUG:teuthology.orchestra.run.trial107:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:37:00.427 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:00.427 DEBUG:teuthology.orchestra.run.trial032:> dd if=/scratch_devs of=/dev/stdout 2026-02-20T21:37:00.472 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-20T21:37:00.472 DEBUG:teuthology.orchestra.run.trial032:> stat /dev/vg_nvme/lv_1 2026-02-20T21:37:00.520 INFO:teuthology.orchestra.run.trial032.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-20T21:37:00.520 INFO:teuthology.orchestra.run.trial032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:37:00.520 INFO:teuthology.orchestra.run.trial032.stdout:Device: 5h/5d Inode: 912 Links: 1 2026-02-20T21:37:00.520 INFO:teuthology.orchestra.run.trial032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:37:00.520 INFO:teuthology.orchestra.run.trial032.stdout:Access: 2026-02-20 21:35:58.122917735 +0000 2026-02-20T21:37:00.520 INFO:teuthology.orchestra.run.trial032.stdout:Modify: 2026-02-20 21:35:57.986920053 +0000 2026-02-20T21:37:00.520 INFO:teuthology.orchestra.run.trial032.stdout:Change: 2026-02-20 21:35:57.986920053 +0000 2026-02-20T21:37:00.520 INFO:teuthology.orchestra.run.trial032.stdout: Birth: - 2026-02-20T21:37:00.521 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-20T21:37:00.572 INFO:teuthology.orchestra.run.trial032.stderr:1+0 records in 2026-02-20T21:37:00.572 INFO:teuthology.orchestra.run.trial032.stderr:1+0 records out 2026-02-20T21:37:00.572 INFO:teuthology.orchestra.run.trial032.stderr:512 bytes copied, 0.000193498 s, 2.6 MB/s 2026-02-20T21:37:00.573 DEBUG:teuthology.orchestra.run.trial032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-20T21:37:00.625 DEBUG:teuthology.orchestra.run.trial032:> stat /dev/vg_nvme/lv_2 2026-02-20T21:37:00.671 INFO:teuthology.orchestra.run.trial032.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-20T21:37:00.671 INFO:teuthology.orchestra.run.trial032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:37:00.672 INFO:teuthology.orchestra.run.trial032.stdout:Device: 5h/5d Inode: 923 Links: 1 2026-02-20T21:37:00.672 INFO:teuthology.orchestra.run.trial032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:37:00.672 INFO:teuthology.orchestra.run.trial032.stdout:Access: 2026-02-20 21:35:58.466911873 +0000 2026-02-20T21:37:00.672 INFO:teuthology.orchestra.run.trial032.stdout:Modify: 2026-02-20 21:35:58.286914941 +0000 2026-02-20T21:37:00.672 INFO:teuthology.orchestra.run.trial032.stdout:Change: 2026-02-20 21:35:58.286914941 +0000 2026-02-20T21:37:00.672 INFO:teuthology.orchestra.run.trial032.stdout: Birth: - 2026-02-20T21:37:00.672 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-20T21:37:00.718 INFO:teuthology.orchestra.run.trial032.stderr:1+0 records in 2026-02-20T21:37:00.719 INFO:teuthology.orchestra.run.trial032.stderr:1+0 records out 2026-02-20T21:37:00.719 INFO:teuthology.orchestra.run.trial032.stderr:512 bytes copied, 0.000181196 s, 2.8 MB/s 2026-02-20T21:37:00.720 DEBUG:teuthology.orchestra.run.trial032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-20T21:37:00.769 DEBUG:teuthology.orchestra.run.trial032:> stat /dev/vg_nvme/lv_3 2026-02-20T21:37:00.815 INFO:teuthology.orchestra.run.trial032.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-20T21:37:00.816 INFO:teuthology.orchestra.run.trial032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:37:00.816 INFO:teuthology.orchestra.run.trial032.stdout:Device: 5h/5d Inode: 946 Links: 1 2026-02-20T21:37:00.816 INFO:teuthology.orchestra.run.trial032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:37:00.816 INFO:teuthology.orchestra.run.trial032.stdout:Access: 2026-02-20 21:35:58.618909283 +0000 2026-02-20T21:37:00.816 INFO:teuthology.orchestra.run.trial032.stdout:Modify: 2026-02-20 21:35:58.614909352 +0000 2026-02-20T21:37:00.816 INFO:teuthology.orchestra.run.trial032.stdout:Change: 2026-02-20 21:35:58.614909352 +0000 2026-02-20T21:37:00.816 INFO:teuthology.orchestra.run.trial032.stdout: Birth: - 2026-02-20T21:37:00.817 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-20T21:37:00.866 INFO:teuthology.orchestra.run.trial032.stderr:1+0 records in 2026-02-20T21:37:00.867 INFO:teuthology.orchestra.run.trial032.stderr:1+0 records out 2026-02-20T21:37:00.867 INFO:teuthology.orchestra.run.trial032.stderr:512 bytes copied, 0.000174292 s, 2.9 MB/s 2026-02-20T21:37:00.868 DEBUG:teuthology.orchestra.run.trial032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-20T21:37:00.917 DEBUG:teuthology.orchestra.run.trial032:> stat /dev/vg_nvme/lv_4 2026-02-20T21:37:00.968 INFO:teuthology.orchestra.run.trial032.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-20T21:37:00.968 INFO:teuthology.orchestra.run.trial032.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:37:00.968 INFO:teuthology.orchestra.run.trial032.stdout:Device: 5h/5d Inode: 971 Links: 1 2026-02-20T21:37:00.968 INFO:teuthology.orchestra.run.trial032.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:37:00.968 INFO:teuthology.orchestra.run.trial032.stdout:Access: 2026-02-20 21:35:58.982903082 +0000 2026-02-20T21:37:00.968 INFO:teuthology.orchestra.run.trial032.stdout:Modify: 2026-02-20 21:35:58.978903149 +0000 2026-02-20T21:37:00.968 INFO:teuthology.orchestra.run.trial032.stdout:Change: 2026-02-20 21:35:58.978903149 +0000 2026-02-20T21:37:00.968 INFO:teuthology.orchestra.run.trial032.stdout: Birth: - 2026-02-20T21:37:00.969 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-20T21:37:01.015 INFO:teuthology.orchestra.run.trial032.stderr:1+0 records in 2026-02-20T21:37:01.015 INFO:teuthology.orchestra.run.trial032.stderr:1+0 records out 2026-02-20T21:37:01.015 INFO:teuthology.orchestra.run.trial032.stderr:512 bytes copied, 0.000170245 s, 3.0 MB/s 2026-02-20T21:37:01.016 DEBUG:teuthology.orchestra.run.trial032:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-20T21:37:01.065 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:37:01.065 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial032.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:37:01.065 INFO:tasks.ceph:Generating config... 2026-02-20T21:37:01.068 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-20T21:37:01.068 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-20T21:37:01.068 INFO:tasks.ceph:[client] log to stderr = False 2026-02-20T21:37:01.068 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-20T21:37:01.068 INFO:tasks.ceph:[client] rbd mirror image policy type = none 2026-02-20T21:37:01.068 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.068 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.069 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.069 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.069 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.069 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.069 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.069 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.069 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-20T21:37:01.069 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-20T21:37:01.069 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-20T21:37:01.069 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-20T21:37:01.069 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-20T21:37:01.070 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-20T21:37:01.070 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-20T21:37:01.070 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-20T21:37:01.070 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-20T21:37:01.070 INFO:tasks.ceph:[osd] bluestore compression algorithm = snappy 2026-02-20T21:37:01.070 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-20T21:37:01.070 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-20T21:37:01.070 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-20T21:37:01.070 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-20T21:37:01.070 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-20T21:37:01.070 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-20T21:37:01.070 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-20T21:37:01.071 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-20T21:37:01.071 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-20T21:37:01.071 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-20T21:37:01.071 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-20T21:37:01.071 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-20T21:37:01.071 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-20T21:37:01.071 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-20T21:37:01.124 INFO:teuthology.orchestra.run.trial032.stdout:creating /etc/ceph/cluster1.keyring 2026-02-20T21:37:01.127 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-20T21:37:01.187 DEBUG:teuthology.orchestra.run.trial032:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-20T21:37:01.235 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.32')] 2026-02-20T21:37:01.236 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.32', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-20T21:37:01.237 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:01.238 DEBUG:teuthology.orchestra.run.trial032:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:37:01.279 DEBUG:teuthology.orchestra.run.trial032:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.32 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:37:01.336 INFO:teuthology.orchestra.run.trial032.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:37:01.337 INFO:teuthology.orchestra.run.trial032.stdout:monmaptool: generated fsid 637d54fe-a5fc-4c51-9045-58e4626774a2 2026-02-20T21:37:01.337 INFO:teuthology.orchestra.run.trial032.stdout:setting min_mon_release = tentacle 2026-02-20T21:37:01.337 INFO:teuthology.orchestra.run.trial032.stdout:epoch 0 2026-02-20T21:37:01.337 INFO:teuthology.orchestra.run.trial032.stdout:fsid 637d54fe-a5fc-4c51-9045-58e4626774a2 2026-02-20T21:37:01.337 INFO:teuthology.orchestra.run.trial032.stdout:last_changed 2026-02-20T21:37:01.337279+0000 2026-02-20T21:37:01.337 INFO:teuthology.orchestra.run.trial032.stdout:created 2026-02-20T21:37:01.337279+0000 2026-02-20T21:37:01.337 INFO:teuthology.orchestra.run.trial032.stdout:min_mon_release 20 (tentacle) 2026-02-20T21:37:01.337 INFO:teuthology.orchestra.run.trial032.stdout:election_strategy: 1 2026-02-20T21:37:01.337 INFO:teuthology.orchestra.run.trial032.stdout:0: [v2:10.20.193.32:3300/0,v1:10.20.193.32:6789/0] mon.a 2026-02-20T21:37:01.338 INFO:teuthology.orchestra.run.trial032.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-20T21:37:01.339 DEBUG:teuthology.orchestra.run.trial032:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:37:01.383 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 637d54fe-a5fc-4c51-9045-58e4626774a2... 2026-02-20T21:37:01.385 DEBUG:teuthology.orchestra.run.trial032:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-20T21:37:01.425 DEBUG:teuthology.orchestra.run.trial107:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-20T21:37:01.442 INFO:teuthology.orchestra.run.trial032.stdout:[global] 2026-02-20T21:37:01.443 INFO:teuthology.orchestra.run.trial032.stdout: chdir = "" 2026-02-20T21:37:01.443 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:37:01.443 INFO:teuthology.orchestra.run.trial032.stdout: auth supported = cephx 2026-02-20T21:37:01.443 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.443 INFO:teuthology.orchestra.run.trial032.stdout: filestore xattr use omap = true 2026-02-20T21:37:01.443 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.443 INFO:teuthology.orchestra.run.trial032.stdout: mon clock drift allowed = 1.000 2026-02-20T21:37:01.443 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.443 INFO:teuthology.orchestra.run.trial032.stdout: osd crush chooseleaf type = 0 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: auth debug = true 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: ms die on old message = true 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: ms die on bug = true 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: mon pg warn max object skew = 0 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.444 INFO:teuthology.orchestra.run.trial032.stdout: osd pool default size = 2 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon osd allow primary affinity = true 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon osd allow pg remap = true 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on no sortbitwise = false 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on too few osds = false 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: mon_allow_pool_size_one = true 2026-02-20T21:37:01.445 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: osd default data pool replay window = 5 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: mon allow pool delete = true 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: mon cluster log file level = debug 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: debug asserts on shutdown = true 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: mon health detail to clog = false 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: mon host = 10.20.193.32 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: mon client directed command retry = 5 2026-02-20T21:37:01.446 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on pool no app = False 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: ms inject socket failures = 5000 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: fsid = 637d54fe-a5fc-4c51-9045-58e4626774a2 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout:[osd] 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: osd journal size = 100 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: osd scrub load threshold = 5.0 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: osd scrub max interval = 600 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: osd mclock skip benchmark = true 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.447 INFO:teuthology.orchestra.run.trial032.stdout: osd recover clone overlap = true 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: osd recovery max chunk = 1048576 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: osd debug shutdown = true 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: osd debug op order = true 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: osd debug verify stray on activate = true 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: osd debug trim objects = true 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: osd open classes on start = true 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: osd debug pg log writeout = true 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.448 INFO:teuthology.orchestra.run.trial032.stdout: osd map max advance = 10 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: journal zero on create = true 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: filestore apply finisher threads = 3 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: bdev debug aio = true 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: osd debug misdirected ops = true 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: bluestore block size = 96636764160 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: bluestore compression algorithm = snappy 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: bluestore compression mode = aggressive 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: bluestore fsck on mount = True 2026-02-20T21:37:01.449 INFO:teuthology.orchestra.run.trial032.stdout: debug bluefs = 1/20 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: debug bluestore = 1/20 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: debug ms = 1 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: debug osd = 20 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: debug rocksdb = 4/10 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: mon osd full ratio = 0.9 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: osd objectstore = bluestore 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout:[mgr] 2026-02-20T21:37:01.450 INFO:teuthology.orchestra.run.trial032.stdout: debug ms = 1 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: debug mgr = 20 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: debug mon = 20 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: debug auth = 20 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: mgr/telemetry/nag = false 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout:[mon] 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: debug ms = 1 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: debug mon = 20 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: debug paxos = 20 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: debug auth = 20 2026-02-20T21:37:01.451 INFO:teuthology.orchestra.run.trial032.stdout: mon data avail warn = 5 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: mon mgr mkfs grace = 240 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: mon osd reporter subtree level = osd 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: mon osd prime pg temp = true 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:37:01.452 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: # 1m isn't quite enough 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout:[client] 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: rgw cache enabled = true 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: rgw enable ops log = true 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: rgw enable usage log = true 2026-02-20T21:37:01.453 INFO:teuthology.orchestra.run.trial032.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: debug rbd = 20 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: debug rbd_mirror = 15 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: log to stderr = False 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: rbd default features = 125 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: rbd mirror image policy type = none 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout:[mon.a] 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.0] 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.1] 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.454 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.455 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.2] 2026-02-20T21:37:01.455 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.455 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.455 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.3] 2026-02-20T21:37:01.455 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.455 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.457 INFO:teuthology.orchestra.run.trial107.stdout:[global] 2026-02-20T21:37:01.457 INFO:teuthology.orchestra.run.trial107.stdout: chdir = "" 2026-02-20T21:37:01.457 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:37:01.457 INFO:teuthology.orchestra.run.trial107.stdout: auth supported = cephx 2026-02-20T21:37:01.457 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.457 INFO:teuthology.orchestra.run.trial107.stdout: filestore xattr use omap = true 2026-02-20T21:37:01.457 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.457 INFO:teuthology.orchestra.run.trial107.stdout: mon clock drift allowed = 1.000 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: osd crush chooseleaf type = 0 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: auth debug = true 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: ms die on old message = true 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: ms die on bug = true 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: mon pg warn max object skew = 0 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:37:01.458 INFO:teuthology.orchestra.run.trial107.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: osd pool default size = 2 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: mon osd allow primary affinity = true 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: mon osd allow pg remap = true 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on no sortbitwise = false 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on too few osds = false 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:37:01.459 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: mon_allow_pool_size_one = true 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: osd default data pool replay window = 5 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: mon allow pool delete = true 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: mon cluster log file level = debug 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: debug asserts on shutdown = true 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: mon health detail to clog = false 2026-02-20T21:37:01.460 INFO:teuthology.orchestra.run.trial107.stdout: mon host = 10.20.193.32 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: mon client directed command retry = 5 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on pool no app = False 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: ms inject socket failures = 5000 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: fsid = 637d54fe-a5fc-4c51-9045-58e4626774a2 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout:[osd] 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: osd journal size = 100 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: osd scrub load threshold = 5.0 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: osd scrub max interval = 600 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: osd mclock skip benchmark = true 2026-02-20T21:37:01.461 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: osd recover clone overlap = true 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: osd recovery max chunk = 1048576 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: osd debug shutdown = true 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: osd debug op order = true 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: osd debug verify stray on activate = true 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: osd debug trim objects = true 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: osd open classes on start = true 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: osd debug pg log writeout = true 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:37:01.462 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: osd map max advance = 10 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: journal zero on create = true 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: filestore apply finisher threads = 3 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: bdev debug aio = true 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: osd debug misdirected ops = true 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: bluestore block size = 96636764160 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: bluestore compression algorithm = snappy 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: bluestore compression mode = aggressive 2026-02-20T21:37:01.463 INFO:teuthology.orchestra.run.trial107.stdout: bluestore fsck on mount = True 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: debug bluefs = 1/20 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: debug bluestore = 1/20 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: debug ms = 1 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: debug osd = 20 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: debug rocksdb = 4/10 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: mon osd full ratio = 0.9 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: osd objectstore = bluestore 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.464 INFO:teuthology.orchestra.run.trial107.stdout:[mgr] 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: debug ms = 1 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: debug mgr = 20 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: debug mon = 20 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: debug auth = 20 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: mgr/telemetry/nag = false 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout:[mon] 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: debug ms = 1 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: debug mon = 20 2026-02-20T21:37:01.465 INFO:teuthology.orchestra.run.trial107.stdout: debug paxos = 20 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: debug auth = 20 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: mon data avail warn = 5 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: mon mgr mkfs grace = 240 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: mon osd reporter subtree level = osd 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: mon osd prime pg temp = true 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.466 INFO:teuthology.orchestra.run.trial107.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: # 1m isn't quite enough 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout:[client] 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: rgw cache enabled = true 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: rgw enable ops log = true 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: rgw enable usage log = true 2026-02-20T21:37:01.467 INFO:teuthology.orchestra.run.trial107.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: debug rbd = 20 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: debug rbd_mirror = 15 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: log to stderr = False 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: rbd default features = 125 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: rbd mirror image policy type = none 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout:[mon.a] 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.0] 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.1] 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.468 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.469 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.2] 2026-02-20T21:37:01.469 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.469 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.469 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.3] 2026-02-20T21:37:01.469 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:01.469 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:01.469 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-20T21:37:01.470 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-20T21:37:01.523 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-20T21:37:01.523 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:01.523 DEBUG:teuthology.orchestra.run.trial032:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-20T21:37:01.568 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:01.568 DEBUG:teuthology.orchestra.run.trial032:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-20T21:37:01.617 INFO:tasks.ceph:Sending monmap to node ubuntu@trial032.front.sepia.ceph.com 2026-02-20T21:37:01.617 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:01.617 DEBUG:teuthology.orchestra.run.trial032:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-20T21:37:01.618 DEBUG:teuthology.orchestra.run.trial032:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-20T21:37:01.673 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:01.674 DEBUG:teuthology.orchestra.run.trial032:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:37:01.720 INFO:tasks.ceph:Sending monmap to node ubuntu@trial107.front.sepia.ceph.com 2026-02-20T21:37:01.720 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:01.720 DEBUG:teuthology.orchestra.run.trial107:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-20T21:37:01.720 DEBUG:teuthology.orchestra.run.trial107:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-20T21:37:01.736 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:01.736 DEBUG:teuthology.orchestra.run.trial107:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:37:01.783 INFO:tasks.ceph:Setting up mon nodes... 2026-02-20T21:37:01.783 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-20T21:37:01.784 DEBUG:teuthology.orchestra.run.trial032:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-20T21:37:01.805 INFO:teuthology.orchestra.run.trial032.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-20T21:37:01.808 INFO:tasks.ceph:Setting up mds nodes... 2026-02-20T21:37:01.808 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-20T21:37:01.808 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-20T21:37:01.874 INFO:teuthology.orchestra.run.trial032.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-20T21:37:01.882 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-20T21:37:01.901 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-20T21:37:01.910 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-20T21:37:01.974 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-20T21:37:01.981 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-20T21:37:02.045 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-20T21:37:02.053 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-20T21:37:02.117 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-20T21:37:02.126 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-20T21:37:02.189 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-20T21:37:02.198 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-20T21:37:02.261 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-20T21:37:02.269 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-20T21:37:02.333 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-20T21:37:02.343 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-20T21:37:02.405 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-20T21:37:02.413 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-20T21:37:02.413 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial032.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:37:02.413 DEBUG:teuthology.orchestra.run.trial032:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-20T21:37:02.423 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:37:02.423 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-20T21:37:02.423 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial032.front.sepia.ceph.com 2026-02-20T21:37:02.423 DEBUG:teuthology.orchestra.run.trial032:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-20T21:37:02.473 INFO:teuthology.orchestra.run.trial032.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:37:02.474 INFO:teuthology.orchestra.run.trial032.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:37:02.474 INFO:teuthology.orchestra.run.trial032.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:37:02.474 INFO:teuthology.orchestra.run.trial032.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-20T21:37:02.474 INFO:teuthology.orchestra.run.trial032.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:37:02.474 INFO:teuthology.orchestra.run.trial032.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:37:02.474 INFO:teuthology.orchestra.run.trial032.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:37:02.474 INFO:teuthology.orchestra.run.trial032.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:37:02.474 INFO:teuthology.orchestra.run.trial032.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:37:02.474 INFO:teuthology.orchestra.run.trial032.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:37:03.047 INFO:teuthology.orchestra.run.trial032.stdout:Discarding blocks...Done. 2026-02-20T21:37:03.048 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial032.front.sepia.ceph.com -o noatime 2026-02-20T21:37:03.049 DEBUG:teuthology.orchestra.run.trial032:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-20T21:37:03.108 DEBUG:teuthology.orchestra.run.trial032:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-20T21:37:03.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:37:03.158 INFO:teuthology.orchestra.run.trial032.stderr:sudo: /sbin/restorecon: command not found 2026-02-20T21:37:03.159 DEBUG:teuthology.orchestra.run.trial032:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-20T21:37:03.208 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:37:03.208 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-20T21:37:03.208 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial032.front.sepia.ceph.com 2026-02-20T21:37:03.209 DEBUG:teuthology.orchestra.run.trial032:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-20T21:37:03.261 INFO:teuthology.orchestra.run.trial032.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:37:03.262 INFO:teuthology.orchestra.run.trial032.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:37:03.262 INFO:teuthology.orchestra.run.trial032.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:37:03.262 INFO:teuthology.orchestra.run.trial032.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-20T21:37:03.262 INFO:teuthology.orchestra.run.trial032.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:37:03.262 INFO:teuthology.orchestra.run.trial032.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:37:03.262 INFO:teuthology.orchestra.run.trial032.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:37:03.262 INFO:teuthology.orchestra.run.trial032.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:37:03.262 INFO:teuthology.orchestra.run.trial032.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:37:03.262 INFO:teuthology.orchestra.run.trial032.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:37:03.826 INFO:teuthology.orchestra.run.trial032.stdout:Discarding blocks...Done. 2026-02-20T21:37:03.828 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial032.front.sepia.ceph.com -o noatime 2026-02-20T21:37:03.828 DEBUG:teuthology.orchestra.run.trial032:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-20T21:37:03.881 DEBUG:teuthology.orchestra.run.trial032:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-20T21:37:03.935 INFO:teuthology.orchestra.run.trial032.stderr:sudo: /sbin/restorecon: command not found 2026-02-20T21:37:03.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:37:03.936 DEBUG:teuthology.orchestra.run.trial032:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-20T21:37:03.987 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:37:03.988 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-20T21:37:03.988 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial032.front.sepia.ceph.com 2026-02-20T21:37:03.988 DEBUG:teuthology.orchestra.run.trial032:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-20T21:37:04.038 INFO:teuthology.orchestra.run.trial032.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:37:04.038 INFO:teuthology.orchestra.run.trial032.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:37:04.038 INFO:teuthology.orchestra.run.trial032.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:37:04.038 INFO:teuthology.orchestra.run.trial032.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-20T21:37:04.038 INFO:teuthology.orchestra.run.trial032.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:37:04.038 INFO:teuthology.orchestra.run.trial032.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:37:04.038 INFO:teuthology.orchestra.run.trial032.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:37:04.038 INFO:teuthology.orchestra.run.trial032.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:37:04.038 INFO:teuthology.orchestra.run.trial032.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:37:04.039 INFO:teuthology.orchestra.run.trial032.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:37:04.605 INFO:teuthology.orchestra.run.trial032.stdout:Discarding blocks...Done. 2026-02-20T21:37:04.607 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial032.front.sepia.ceph.com -o noatime 2026-02-20T21:37:04.607 DEBUG:teuthology.orchestra.run.trial032:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-20T21:37:04.665 DEBUG:teuthology.orchestra.run.trial032:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-20T21:37:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:37:04.714 INFO:teuthology.orchestra.run.trial032.stderr:sudo: /sbin/restorecon: command not found 2026-02-20T21:37:04.715 DEBUG:teuthology.orchestra.run.trial032:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:37:04.777 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:04.774+0000 7fd794595a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-20T21:37:04.778 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:04.774+0000 7fd794595a40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-20T21:37:04.778 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:04.774+0000 7fd794595a40 -1 bdev(0x55f793037800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-20T21:37:04.779 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:04.774+0000 7fd794595a40 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-20T21:37:05.696 DEBUG:teuthology.orchestra.run.trial032:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-20T21:37:05.749 DEBUG:teuthology.orchestra.run.trial032:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:37:05.817 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:05.814+0000 7f1dd2aaea40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-20T21:37:05.817 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:05.814+0000 7f1dd2aaea40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-20T21:37:05.818 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:05.814+0000 7f1dd2aaea40 -1 bdev(0x5631d1153800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-20T21:37:05.818 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:05.814+0000 7f1dd2aaea40 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-20T21:37:06.720 DEBUG:teuthology.orchestra.run.trial032:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-20T21:37:06.769 DEBUG:teuthology.orchestra.run.trial032:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:37:06.833 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:06.830+0000 7f1fef10aa40 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-20T21:37:06.834 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:06.830+0000 7f1fef10aa40 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-20T21:37:06.834 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:06.830+0000 7f1fef10aa40 -1 bdev(0x564bbb00b800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-20T21:37:06.834 INFO:teuthology.orchestra.run.trial032.stderr:2026-02-20T21:37:06.830+0000 7f1fef10aa40 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-20T21:37:07.673 DEBUG:teuthology.orchestra.run.trial032:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-20T21:37:07.725 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-20T21:37:07.725 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:07.725 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-20T21:37:07.777 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:07.777 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-20T21:37:07.829 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:07.829 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-20T21:37:07.881 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:07.881 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-20T21:37:07.933 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:07.933 DEBUG:teuthology.orchestra.run.trial032:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-20T21:37:07.980 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:07.980 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-20T21:37:07.986 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:07.986 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-20T21:37:08.031 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:08.031 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-20T21:37:08.079 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:08.080 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-20T21:37:08.127 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:08.127 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-20T21:37:08.175 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:08.176 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-20T21:37:08.226 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:08.226 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-20T21:37:08.271 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:08.271 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-20T21:37:08.319 INFO:tasks.ceph:Adding keys to all mons... 2026-02-20T21:37:08.319 DEBUG:teuthology.orchestra.run.trial032:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-20T21:37:08.327 INFO:teuthology.orchestra.run.trial032.stdout:[mgr.x] 2026-02-20T21:37:08.328 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD905hpaLARMBAAYhYoohkLj80sbpr8TCvA/w== 2026-02-20T21:37:08.328 INFO:teuthology.orchestra.run.trial032.stdout:[osd.0] 2026-02-20T21:37:08.328 INFO:teuthology.orchestra.run.trial032.stdout: key = AQAA1Jhp0mJlLhAAwAYr3toX7P+pP2w9Vss+cw== 2026-02-20T21:37:08.328 INFO:teuthology.orchestra.run.trial032.stdout:[osd.1] 2026-02-20T21:37:08.328 INFO:teuthology.orchestra.run.trial032.stdout: key = AQAB1JhpuD++MBAAzPSqUWJyXLBae2BEHwi1SA== 2026-02-20T21:37:08.328 INFO:teuthology.orchestra.run.trial032.stdout:[osd.2] 2026-02-20T21:37:08.328 INFO:teuthology.orchestra.run.trial032.stdout: key = AQAC1JhpL966MRAAW6HVmbon8iwUXe/FDSuN2w== 2026-02-20T21:37:08.328 INFO:teuthology.orchestra.run.trial032.stdout:[client.0] 2026-02-20T21:37:08.328 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD905hpQ9MwNBAAnzp7KxBK4kf+KZvbKWJhDA== 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror] 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD905hpMNaaNRAAw3rq8pjpJ39mjt21pn2xnQ== 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.0] 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD905hpL23qORAA2psowlCFtu8YiflrIiLn1Q== 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.1] 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD+05hpfcORAhAAoxh71EL4vMj2w6djXiLd9g== 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.2] 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD+05hpsljbBhAAgEy41iZa4fAxkijHfKxJ4A== 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.3] 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD+05hp4aEjCxAAE3KBsUxhRKYmTAHoTZF0ZA== 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.4] 2026-02-20T21:37:08.329 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD+05hpjthwDxAAmcmFNqHTXWJmeEykilRUMA== 2026-02-20T21:37:08.330 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.5] 2026-02-20T21:37:08.330 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD+05hpv9y9ExAAylsCZy7GIyftYVu/iYTp+A== 2026-02-20T21:37:08.330 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.6] 2026-02-20T21:37:08.330 INFO:teuthology.orchestra.run.trial032.stdout: key = AQD+05hpLBMJGBAAEO3BXREXgVtHbpJn9vywSQ== 2026-02-20T21:37:08.331 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-20T21:37:08.399 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:37:08.462 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:37:08.528 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:37:08.590 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:08.654 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:08.718 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:08.782 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:08.846 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:08.910 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:08.974 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:09.038 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:09.102 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:09.166 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-20T21:37:09.166 DEBUG:teuthology.orchestra.run.trial032:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-20T21:37:09.216 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-20T21:37:09.286 DEBUG:teuthology.orchestra.run.trial032:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-20T21:37:09.336 DEBUG:teuthology.orchestra.run.trial032:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:37:09.383 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-20T21:37:09.384 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-20T21:37:09.384 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-20T21:37:09.425 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-20T21:37:09.426 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-20T21:37:09.426 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-20T21:37:09.426 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-20T21:37:09.428 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-20T21:37:09.429 DEBUG:tasks.ceph:set 0 configs 2026-02-20T21:37:09.429 DEBUG:teuthology.orchestra.run.trial032:> sudo ceph --cluster cluster1 config dump 2026-02-20T21:37:09.545 INFO:teuthology.orchestra.run.trial032.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-20T21:37:09.557 INFO:tasks.ceph:Setting crush tunables to default 2026-02-20T21:37:09.558 DEBUG:teuthology.orchestra.run.trial032:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-20T21:37:09.655 INFO:teuthology.orchestra.run.trial032.stderr:adjusted tunables profile to default 2026-02-20T21:37:09.668 INFO:tasks.ceph:check_enable_crimson: False 2026-02-20T21:37:09.668 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-20T21:37:09.669 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:09.669 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-20T21:37:09.679 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:09.679 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-20T21:37:09.728 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:09.728 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-20T21:37:09.775 DEBUG:teuthology.orchestra.run.trial032:> sudo ceph --cluster cluster1 osd new d4d665db-e2ee-45c1-a7d0-483f617a59af 0 2026-02-20T21:37:09.912 INFO:teuthology.orchestra.run.trial032.stdout:0 2026-02-20T21:37:09.922 DEBUG:teuthology.orchestra.run.trial032:> sudo ceph --cluster cluster1 osd new f94266a3-dac8-4ba7-b0be-47a485fc51d0 1 2026-02-20T21:37:10.020 INFO:teuthology.orchestra.run.trial032.stdout:1 2026-02-20T21:37:10.030 DEBUG:teuthology.orchestra.run.trial032:> sudo ceph --cluster cluster1 osd new 15a644f0-20e2-46b2-9a46-894f8dcf405c 2 2026-02-20T21:37:10.129 INFO:teuthology.orchestra.run.trial032.stdout:2 2026-02-20T21:37:10.139 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-20T21:37:10.139 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-20T21:37:10.141 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-20T21:37:10.141 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-20T21:37:10.141 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-20T21:37:10.143 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-20T21:37:10.143 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-20T21:37:10.143 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-20T21:37:10.145 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-20T21:37:10.146 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:37:10.247 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:10.247 INFO:teuthology.orchestra.run.trial032.stdout:{"epoch":5,"fsid":"637d54fe-a5fc-4c51-9045-58e4626774a2","created":"2026-02-20T21:37:09.505649+0000","modified":"2026-02-20T21:37:10.129240+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:37:10.129240+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"d4d665db-e2ee-45c1-a7d0-483f617a59af","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"f94266a3-dac8-4ba7-b0be-47a485fc51d0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"15a644f0-20e2-46b2-9a46-894f8dcf405c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:10.257 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-20T21:37:10.257 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-20T21:37:10.559 DEBUG:teuthology.orchestra.run.trial032:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:37:10.689 INFO:tasks.ceph.cluster1.osd.0.trial032.stderr:2026-02-20T21:37:10.686+0000 7f2f11f70a40 -1 Falling back to public interface 2026-02-20T21:37:10.741 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:37:10.738+0000 7f1b55f63a40 -1 Falling back to public interface 2026-02-20T21:37:10.757 INFO:tasks.ceph.cluster1.osd.2.trial032.stderr:2026-02-20T21:37:10.754+0000 7f806520da40 -1 Falling back to public interface 2026-02-20T21:37:10.855 INFO:teuthology.misc.health.trial032.stdout: 2026-02-20T21:37:10.855 INFO:teuthology.misc.health.trial032.stdout:{"epoch":5,"fsid":"637d54fe-a5fc-4c51-9045-58e4626774a2","created":"2026-02-20T21:37:09.505649+0000","modified":"2026-02-20T21:37:10.129240+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:37:10.129240+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"d4d665db-e2ee-45c1-a7d0-483f617a59af","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"f94266a3-dac8-4ba7-b0be-47a485fc51d0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"15a644f0-20e2-46b2-9a46-894f8dcf405c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:10.864 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-20T21:37:11.018 INFO:tasks.ceph.cluster1.mgr.x.trial032.stderr:ERROR:root:Module 'jmespath' is not installed. 2026-02-20T21:37:11.071 INFO:tasks.ceph.cluster1.mgr.x.trial032.stderr:ERROR:dashboard.controllers.nvmeof:Failed to import NVMeoFClient and related components: cannot import name 'NVMeoFClient' from 'dashboard.services.nvmeof_client' (/usr/share/ceph/mgr/dashboard/services/nvmeof_client.py) 2026-02-20T21:37:11.076 INFO:tasks.ceph.cluster1.mgr.x.trial032.stderr:ERROR:sso:Module 'jmespath' is not installed. 2026-02-20T21:37:11.123 INFO:tasks.ceph.cluster1.mgr.x.trial032.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-20T21:37:11.546 INFO:tasks.ceph.cluster1.mgr.x.trial032.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-20T21:37:11.546 INFO:tasks.ceph.cluster1.mgr.x.trial032.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-20T21:37:11.546 INFO:tasks.ceph.cluster1.mgr.x.trial032.stderr: from numpy import show_config as show_numpy_config 2026-02-20T21:37:11.973 INFO:tasks.ceph.cluster1.osd.0.trial032.stderr:2026-02-20T21:37:11.970+0000 7f2f11f70a40 -1 osd.0 0 log_to_monitors true 2026-02-20T21:37:12.114 INFO:tasks.ceph.cluster1.osd.2.trial032.stderr:2026-02-20T21:37:12.110+0000 7f806520da40 -1 osd.2 0 log_to_monitors true 2026-02-20T21:37:12.119 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:37:12.114+0000 7f1b55f63a40 -1 osd.1 0 log_to_monitors true 2026-02-20T21:37:12.621 INFO:tasks.ceph.cluster1.mgr.x.trial032.stderr:2026-02-20T21:37:12.618+0000 7f72ee2c9640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-20T21:37:13.668 INFO:tasks.ceph.cluster1.osd.0.trial032.stderr:2026-02-20T21:37:13.666+0000 7f2f0df19640 -1 osd.0 0 waiting for initial osdmap 2026-02-20T21:37:13.669 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:37:13.666+0000 7f1b51f0c640 -1 osd.1 0 waiting for initial osdmap 2026-02-20T21:37:13.669 INFO:tasks.ceph.cluster1.osd.2.trial032.stderr:2026-02-20T21:37:13.666+0000 7f80611b6640 -1 osd.2 0 waiting for initial osdmap 2026-02-20T21:37:13.671 INFO:tasks.ceph.cluster1.osd.2.trial032.stderr:2026-02-20T21:37:13.666+0000 7f80587bd640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:37:13.671 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:37:13.666+0000 7f1b49513640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:37:13.672 INFO:tasks.ceph.cluster1.osd.0.trial032.stderr:2026-02-20T21:37:13.666+0000 7f2f05520640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:37:17.167 DEBUG:teuthology.orchestra.run.trial032:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:37:17.313 INFO:teuthology.misc.health.trial032.stdout: 2026-02-20T21:37:17.313 INFO:teuthology.misc.health.trial032.stdout:{"epoch":10,"fsid":"637d54fe-a5fc-4c51-9045-58e4626774a2","created":"2026-02-20T21:37:09.505649+0000","modified":"2026-02-20T21:37:16.658753+0000","last_up_change":"2026-02-20T21:37:14.652696+0000","last_in_change":"2026-02-20T21:37:10.129240+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:37:15.649121+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d4d665db-e2ee-45c1-a7d0-483f617a59af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6800","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6801","nonce":2386430400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6802","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6803","nonce":2386430400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6806","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6807","nonce":2386430400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6804","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6805","nonce":2386430400}]},"public_addr":"10.20.193.32:6801/2386430400","cluster_addr":"10.20.193.32:6803/2386430400","heartbeat_back_addr":"10.20.193.32:6807/2386430400","heartbeat_front_addr":"10.20.193.32:6805/2386430400","state":["exists","up"]},{"osd":1,"uuid":"f94266a3-dac8-4ba7-b0be-47a485fc51d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6808","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6809","nonce":128603091}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6810","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6811","nonce":128603091}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6814","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6815","nonce":128603091}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6812","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6813","nonce":128603091}]},"public_addr":"10.20.193.32:6809/128603091","cluster_addr":"10.20.193.32:6811/128603091","heartbeat_back_addr":"10.20.193.32:6815/128603091","heartbeat_front_addr":"10.20.193.32:6813/128603091","state":["exists","up"]},{"osd":2,"uuid":"15a644f0-20e2-46b2-9a46-894f8dcf405c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6816","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6817","nonce":2752957932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6818","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6819","nonce":2752957932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6822","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6823","nonce":2752957932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6820","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6821","nonce":2752957932}]},"public_addr":"10.20.193.32:6817/2752957932","cluster_addr":"10.20.193.32:6819/2752957932","heartbeat_back_addr":"10.20.193.32:6823/2752957932","heartbeat_front_addr":"10.20.193.32:6821/2752957932","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:17.324 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-20T21:37:17.324 INFO:tasks.ceph:Creating RBD pool 2026-02-20T21:37:17.324 DEBUG:teuthology.orchestra.run.trial032:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-20T21:37:17.662 INFO:teuthology.orchestra.run.trial032.stderr:pool 'rbd' created 2026-02-20T21:37:17.675 DEBUG:teuthology.orchestra.run.trial032:> rbd --cluster cluster1 pool init rbd 2026-02-20T21:37:20.677 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-20T21:37:20.678 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2026-02-20T21:37:20.679 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-20T21:37:20.837 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:20.847 INFO:teuthology.orchestra.run.trial032.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-20T21:37:20.848 INFO:tasks.ceph_manager:config epoch is 1 2026-02-20T21:37:20.848 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-20T21:37:20.848 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-20T21:37:20.848 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-20T21:37:21.022 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:21.032 INFO:teuthology.orchestra.run.trial032.stdout:{"epoch":5,"flags":0,"active_gid":4104,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6824","nonce":812862852},{"type":"v1","addr":"10.20.193.32:6825","nonce":812862852}]},"active_addr":"10.20.193.32:6825/812862852","active_change":"2026-02-20T21:37:11.613682+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.32:0","nonce":3778086935}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.32:0","nonce":1235780202}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.32:0","nonce":2099313076}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.32:0","nonce":3350781982}]}]} 2026-02-20T21:37:21.036 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-20T21:37:21.036 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-20T21:37:21.036 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:37:21.187 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:21.188 INFO:teuthology.orchestra.run.trial032.stdout:{"epoch":14,"fsid":"637d54fe-a5fc-4c51-9045-58e4626774a2","created":"2026-02-20T21:37:09.505649+0000","modified":"2026-02-20T21:37:20.666651+0000","last_up_change":"2026-02-20T21:37:14.652696+0000","last_in_change":"2026-02-20T21:37:10.129240+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:37:15.649121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:37:17.481675+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d4d665db-e2ee-45c1-a7d0-483f617a59af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6800","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6801","nonce":2386430400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6802","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6803","nonce":2386430400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6806","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6807","nonce":2386430400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6804","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6805","nonce":2386430400}]},"public_addr":"10.20.193.32:6801/2386430400","cluster_addr":"10.20.193.32:6803/2386430400","heartbeat_back_addr":"10.20.193.32:6807/2386430400","heartbeat_front_addr":"10.20.193.32:6805/2386430400","state":["exists","up"]},{"osd":1,"uuid":"f94266a3-dac8-4ba7-b0be-47a485fc51d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6808","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6809","nonce":128603091}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6810","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6811","nonce":128603091}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6814","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6815","nonce":128603091}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6812","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6813","nonce":128603091}]},"public_addr":"10.20.193.32:6809/128603091","cluster_addr":"10.20.193.32:6811/128603091","heartbeat_back_addr":"10.20.193.32:6815/128603091","heartbeat_front_addr":"10.20.193.32:6813/128603091","state":["exists","up"]},{"osd":2,"uuid":"15a644f0-20e2-46b2-9a46-894f8dcf405c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6816","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6817","nonce":2752957932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6818","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6819","nonce":2752957932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6822","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6823","nonce":2752957932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6820","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6821","nonce":2752957932}]},"public_addr":"10.20.193.32:6817/2752957932","cluster_addr":"10.20.193.32:6819/2752957932","heartbeat_back_addr":"10.20.193.32:6823/2752957932","heartbeat_front_addr":"10.20.193.32:6821/2752957932","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:21.200 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-20T21:37:21.200 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:37:21.362 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:21.362 INFO:teuthology.orchestra.run.trial032.stdout:{"epoch":14,"fsid":"637d54fe-a5fc-4c51-9045-58e4626774a2","created":"2026-02-20T21:37:09.505649+0000","modified":"2026-02-20T21:37:20.666651+0000","last_up_change":"2026-02-20T21:37:14.652696+0000","last_in_change":"2026-02-20T21:37:10.129240+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:37:15.649121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:37:17.481675+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d4d665db-e2ee-45c1-a7d0-483f617a59af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6800","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6801","nonce":2386430400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6802","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6803","nonce":2386430400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6806","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6807","nonce":2386430400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6804","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6805","nonce":2386430400}]},"public_addr":"10.20.193.32:6801/2386430400","cluster_addr":"10.20.193.32:6803/2386430400","heartbeat_back_addr":"10.20.193.32:6807/2386430400","heartbeat_front_addr":"10.20.193.32:6805/2386430400","state":["exists","up"]},{"osd":1,"uuid":"f94266a3-dac8-4ba7-b0be-47a485fc51d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6808","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6809","nonce":128603091}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6810","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6811","nonce":128603091}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6814","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6815","nonce":128603091}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6812","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6813","nonce":128603091}]},"public_addr":"10.20.193.32:6809/128603091","cluster_addr":"10.20.193.32:6811/128603091","heartbeat_back_addr":"10.20.193.32:6815/128603091","heartbeat_front_addr":"10.20.193.32:6813/128603091","state":["exists","up"]},{"osd":2,"uuid":"15a644f0-20e2-46b2-9a46-894f8dcf405c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6816","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6817","nonce":2752957932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6818","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6819","nonce":2752957932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6822","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6823","nonce":2752957932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6820","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6821","nonce":2752957932}]},"public_addr":"10.20.193.32:6817/2752957932","cluster_addr":"10.20.193.32:6819/2752957932","heartbeat_back_addr":"10.20.193.32:6823/2752957932","heartbeat_front_addr":"10.20.193.32:6821/2752957932","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:21.373 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-20T21:37:21.374 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-20T21:37:21.374 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-20T21:37:21.461 INFO:teuthology.orchestra.run.trial032.stdout:34359738372 2026-02-20T21:37:21.461 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-20T21:37:21.463 INFO:teuthology.orchestra.run.trial032.stdout:34359738372 2026-02-20T21:37:21.463 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-20T21:37:21.464 INFO:teuthology.orchestra.run.trial032.stdout:34359738371 2026-02-20T21:37:21.465 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-20T21:37:21.659 INFO:teuthology.orchestra.run.trial032.stdout:34359738370 2026-02-20T21:37:21.661 INFO:teuthology.orchestra.run.trial032.stdout:34359738369 2026-02-20T21:37:21.668 INFO:teuthology.orchestra.run.trial032.stdout:34359738370 2026-02-20T21:37:21.672 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738370 for osd.2 2026-02-20T21:37:21.673 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738369 for osd.0 2026-02-20T21:37:21.680 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738370 for osd.1 2026-02-20T21:37:22.672 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-20T21:37:22.673 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-20T21:37:22.681 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-20T21:37:22.828 INFO:teuthology.orchestra.run.trial032.stdout:34359738371 2026-02-20T21:37:22.835 INFO:teuthology.orchestra.run.trial032.stdout:34359738372 2026-02-20T21:37:22.842 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.0 2026-02-20T21:37:22.842 DEBUG:teuthology.parallel:result is None 2026-02-20T21:37:22.848 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.2 2026-02-20T21:37:22.848 DEBUG:teuthology.parallel:result is None 2026-02-20T21:37:22.888 INFO:teuthology.orchestra.run.trial032.stdout:34359738372 2026-02-20T21:37:22.899 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738372 for osd.1 2026-02-20T21:37:22.900 DEBUG:teuthology.parallel:result is None 2026-02-20T21:37:22.900 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-20T21:37:22.900 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:37:23.093 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:23.094 INFO:teuthology.orchestra.run.trial032.stderr:dumped all 2026-02-20T21:37:23.103 INFO:teuthology.orchestra.run.trial032.stdout:{"pg_ready":true,"pg_map":{"version":15,"stamp":"2026-02-20T21:37:21.618109+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":12,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81388,"kb_used_data":496,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283034132,"statfs":{"total":289910292480,"available":289826951168,"internally_reserved":0,"allocated":507904,"data_stored":1287022,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":23725,"internal_metadata":82748243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.950666"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673303+0000","last_change":"2026-02-20T21:37:20.673446+0000","last_active":"2026-02-20T21:37:20.673303+0000","last_peered":"2026-02-20T21:37:20.673303+0000","last_clean":"2026-02-20T21:37:20.673303+0000","last_became_active":"2026-02-20T21:37:18.668090+0000","last_became_peered":"2026-02-20T21:37:18.668090+0000","last_unstale":"2026-02-20T21:37:20.673303+0000","last_undegraded":"2026-02-20T21:37:20.673303+0000","last_fullsized":"2026-02-20T21:37:20.673303+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:30:26.985444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038342900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673473+0000","last_change":"2026-02-20T21:37:20.673605+0000","last_active":"2026-02-20T21:37:20.673473+0000","last_peered":"2026-02-20T21:37:20.673473+0000","last_clean":"2026-02-20T21:37:20.673473+0000","last_became_active":"2026-02-20T21:37:18.668149+0000","last_became_peered":"2026-02-20T21:37:18.668149+0000","last_unstale":"2026-02-20T21:37:20.673473+0000","last_undegraded":"2026-02-20T21:37:20.673473+0000","last_fullsized":"2026-02-20T21:37:20.673473+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:13:26.573716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055130999999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673650+0000","last_change":"2026-02-20T21:37:20.673738+0000","last_active":"2026-02-20T21:37:20.673650+0000","last_peered":"2026-02-20T21:37:20.673650+0000","last_clean":"2026-02-20T21:37:20.673650+0000","last_became_active":"2026-02-20T21:37:18.667922+0000","last_became_peered":"2026-02-20T21:37:18.667922+0000","last_unstale":"2026-02-20T21:37:20.673650+0000","last_undegraded":"2026-02-20T21:37:20.673650+0000","last_fullsized":"2026-02-20T21:37:20.673650+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:19:51.832309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065968000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673497+0000","last_change":"2026-02-20T21:37:20.673640+0000","last_active":"2026-02-20T21:37:20.673497+0000","last_peered":"2026-02-20T21:37:20.673497+0000","last_clean":"2026-02-20T21:37:20.673497+0000","last_became_active":"2026-02-20T21:37:18.668675+0000","last_became_peered":"2026-02-20T21:37:18.668675+0000","last_unstale":"2026-02-20T21:37:20.673497+0000","last_undegraded":"2026-02-20T21:37:20.673497+0000","last_fullsized":"2026-02-20T21:37:20.673497+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:14:58.246769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058640500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.671740+0000","last_change":"2026-02-20T21:37:20.671740+0000","last_active":"2026-02-20T21:37:20.671740+0000","last_peered":"2026-02-20T21:37:20.671740+0000","last_clean":"2026-02-20T21:37:20.671740+0000","last_became_active":"2026-02-20T21:37:18.668118+0000","last_became_peered":"2026-02-20T21:37:18.668118+0000","last_unstale":"2026-02-20T21:37:20.671740+0000","last_undegraded":"2026-02-20T21:37:20.671740+0000","last_fullsized":"2026-02-20T21:37:20.671740+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:38:22.882998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068900500000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.668690+0000","last_change":"2026-02-20T21:37:20.668922+0000","last_active":"2026-02-20T21:37:20.668690+0000","last_peered":"2026-02-20T21:37:20.668690+0000","last_clean":"2026-02-20T21:37:20.668690+0000","last_became_active":"2026-02-20T21:37:18.668095+0000","last_became_peered":"2026-02-20T21:37:18.668095+0000","last_unstale":"2026-02-20T21:37:20.668690+0000","last_undegraded":"2026-02-20T21:37:20.668690+0000","last_fullsized":"2026-02-20T21:37:20.668690+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:26:41.579853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045394000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.668646+0000","last_change":"2026-02-20T21:37:20.668873+0000","last_active":"2026-02-20T21:37:20.668646+0000","last_peered":"2026-02-20T21:37:20.668646+0000","last_clean":"2026-02-20T21:37:20.668646+0000","last_became_active":"2026-02-20T21:37:18.668650+0000","last_became_peered":"2026-02-20T21:37:18.668650+0000","last_unstale":"2026-02-20T21:37:20.668646+0000","last_undegraded":"2026-02-20T21:37:20.668646+0000","last_fullsized":"2026-02-20T21:37:20.668646+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:09:53.022016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040528899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673273+0000","last_change":"2026-02-20T21:37:20.673439+0000","last_active":"2026-02-20T21:37:20.673273+0000","last_peered":"2026-02-20T21:37:20.673273+0000","last_clean":"2026-02-20T21:37:20.673273+0000","last_became_active":"2026-02-20T21:37:18.667168+0000","last_became_peered":"2026-02-20T21:37:18.667168+0000","last_unstale":"2026-02-20T21:37:20.673273+0000","last_undegraded":"2026-02-20T21:37:20.673273+0000","last_fullsized":"2026-02-20T21:37:20.673273+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:25:34.870707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037955200000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673040+0000","last_change":"2026-02-20T21:37:16.895581+0000","last_active":"2026-02-20T21:37:20.673040+0000","last_peered":"2026-02-20T21:37:20.673040+0000","last_clean":"2026-02-20T21:37:20.673040+0000","last_became_active":"2026-02-20T21:37:16.894669+0000","last_became_peered":"2026-02-20T21:37:16.894669+0000","last_unstale":"2026-02-20T21:37:20.673040+0000","last_undegraded":"2026-02-20T21:37:20.673040+0000","last_fullsized":"2026-02-20T21:37:20.673040+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_clean_scrub_stamp":"2026-02-20T21:37:15.654907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:21:36.071811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":35416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":625803,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27160,"kb_used_data":188,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344680,"statfs":{"total":96636764160,"available":96608952320,"internally_reserved":0,"allocated":192512,"data_stored":625803,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:37:23.105 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:37:23.262 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:23.262 INFO:teuthology.orchestra.run.trial032.stderr:dumped all 2026-02-20T21:37:23.272 INFO:teuthology.orchestra.run.trial032.stdout:{"pg_ready":true,"pg_map":{"version":15,"stamp":"2026-02-20T21:37:21.618109+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":12,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81388,"kb_used_data":496,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283034132,"statfs":{"total":289910292480,"available":289826951168,"internally_reserved":0,"allocated":507904,"data_stored":1287022,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":23725,"internal_metadata":82748243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.950666"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673303+0000","last_change":"2026-02-20T21:37:20.673446+0000","last_active":"2026-02-20T21:37:20.673303+0000","last_peered":"2026-02-20T21:37:20.673303+0000","last_clean":"2026-02-20T21:37:20.673303+0000","last_became_active":"2026-02-20T21:37:18.668090+0000","last_became_peered":"2026-02-20T21:37:18.668090+0000","last_unstale":"2026-02-20T21:37:20.673303+0000","last_undegraded":"2026-02-20T21:37:20.673303+0000","last_fullsized":"2026-02-20T21:37:20.673303+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:30:26.985444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038342900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673473+0000","last_change":"2026-02-20T21:37:20.673605+0000","last_active":"2026-02-20T21:37:20.673473+0000","last_peered":"2026-02-20T21:37:20.673473+0000","last_clean":"2026-02-20T21:37:20.673473+0000","last_became_active":"2026-02-20T21:37:18.668149+0000","last_became_peered":"2026-02-20T21:37:18.668149+0000","last_unstale":"2026-02-20T21:37:20.673473+0000","last_undegraded":"2026-02-20T21:37:20.673473+0000","last_fullsized":"2026-02-20T21:37:20.673473+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:13:26.573716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055130999999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673650+0000","last_change":"2026-02-20T21:37:20.673738+0000","last_active":"2026-02-20T21:37:20.673650+0000","last_peered":"2026-02-20T21:37:20.673650+0000","last_clean":"2026-02-20T21:37:20.673650+0000","last_became_active":"2026-02-20T21:37:18.667922+0000","last_became_peered":"2026-02-20T21:37:18.667922+0000","last_unstale":"2026-02-20T21:37:20.673650+0000","last_undegraded":"2026-02-20T21:37:20.673650+0000","last_fullsized":"2026-02-20T21:37:20.673650+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:19:51.832309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065968000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673497+0000","last_change":"2026-02-20T21:37:20.673640+0000","last_active":"2026-02-20T21:37:20.673497+0000","last_peered":"2026-02-20T21:37:20.673497+0000","last_clean":"2026-02-20T21:37:20.673497+0000","last_became_active":"2026-02-20T21:37:18.668675+0000","last_became_peered":"2026-02-20T21:37:18.668675+0000","last_unstale":"2026-02-20T21:37:20.673497+0000","last_undegraded":"2026-02-20T21:37:20.673497+0000","last_fullsized":"2026-02-20T21:37:20.673497+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:14:58.246769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058640500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.671740+0000","last_change":"2026-02-20T21:37:20.671740+0000","last_active":"2026-02-20T21:37:20.671740+0000","last_peered":"2026-02-20T21:37:20.671740+0000","last_clean":"2026-02-20T21:37:20.671740+0000","last_became_active":"2026-02-20T21:37:18.668118+0000","last_became_peered":"2026-02-20T21:37:18.668118+0000","last_unstale":"2026-02-20T21:37:20.671740+0000","last_undegraded":"2026-02-20T21:37:20.671740+0000","last_fullsized":"2026-02-20T21:37:20.671740+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:38:22.882998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068900500000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.668690+0000","last_change":"2026-02-20T21:37:20.668922+0000","last_active":"2026-02-20T21:37:20.668690+0000","last_peered":"2026-02-20T21:37:20.668690+0000","last_clean":"2026-02-20T21:37:20.668690+0000","last_became_active":"2026-02-20T21:37:18.668095+0000","last_became_peered":"2026-02-20T21:37:18.668095+0000","last_unstale":"2026-02-20T21:37:20.668690+0000","last_undegraded":"2026-02-20T21:37:20.668690+0000","last_fullsized":"2026-02-20T21:37:20.668690+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:26:41.579853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045394000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.668646+0000","last_change":"2026-02-20T21:37:20.668873+0000","last_active":"2026-02-20T21:37:20.668646+0000","last_peered":"2026-02-20T21:37:20.668646+0000","last_clean":"2026-02-20T21:37:20.668646+0000","last_became_active":"2026-02-20T21:37:18.668650+0000","last_became_peered":"2026-02-20T21:37:18.668650+0000","last_unstale":"2026-02-20T21:37:20.668646+0000","last_undegraded":"2026-02-20T21:37:20.668646+0000","last_fullsized":"2026-02-20T21:37:20.668646+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:09:53.022016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040528899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673273+0000","last_change":"2026-02-20T21:37:20.673439+0000","last_active":"2026-02-20T21:37:20.673273+0000","last_peered":"2026-02-20T21:37:20.673273+0000","last_clean":"2026-02-20T21:37:20.673273+0000","last_became_active":"2026-02-20T21:37:18.667168+0000","last_became_peered":"2026-02-20T21:37:18.667168+0000","last_unstale":"2026-02-20T21:37:20.673273+0000","last_undegraded":"2026-02-20T21:37:20.673273+0000","last_fullsized":"2026-02-20T21:37:20.673273+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:25:34.870707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037955200000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:20.673040+0000","last_change":"2026-02-20T21:37:16.895581+0000","last_active":"2026-02-20T21:37:20.673040+0000","last_peered":"2026-02-20T21:37:20.673040+0000","last_clean":"2026-02-20T21:37:20.673040+0000","last_became_active":"2026-02-20T21:37:16.894669+0000","last_became_peered":"2026-02-20T21:37:16.894669+0000","last_unstale":"2026-02-20T21:37:20.673040+0000","last_undegraded":"2026-02-20T21:37:20.673040+0000","last_fullsized":"2026-02-20T21:37:20.673040+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_clean_scrub_stamp":"2026-02-20T21:37:15.654907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:21:36.071811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":35416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":625803,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27160,"kb_used_data":188,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344680,"statfs":{"total":96636764160,"available":96608952320,"internally_reserved":0,"allocated":192512,"data_stored":625803,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:37:23.273 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-20T21:37:23.273 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-20T21:37:23.273 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-20T21:37:23.274 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-20T21:37:23.451 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:23.451 INFO:teuthology.orchestra.run.trial032.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-20T21:37:23.463 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-20T21:37:23.464 INFO:teuthology.run_tasks:Running task ceph... 2026-02-20T21:37:23.473 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-20T21:37:23.474 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-20T21:37:23.474 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486', 'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-snappy policy/none rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-stress-workunit-fast-diff}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '61486', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial', 'no_nested_subset': False, 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7161, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f7870454149f22574086dd2ff55d050585136531', 'targets': {'trial032.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPRXC6A0nz0Ujzi5yHHzygV4NFoxUEbgNW5j0qSKAWKxFkkbu/brPWlpCdqn6tyLIGv6fhrzujk2pJ6qXpMi07s=', 'trial107.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBG0Ox7QcjWoqAVtqdwVfCJ6pcSiRRJmVoXtIhl0//qNUjkwWwsjgx3aj2wx7whWZfyuqAMUwwncaoPTMGgKoBzo='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_IMAGE_FEATURES': 'layering,exclusive-lock,object-map,fast-diff', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-20_16:15:01', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699'} 2026-02-20T21:37:23.475 DEBUG:teuthology.orchestra.run.trial032:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-20T21:37:23.477 DEBUG:teuthology.orchestra.run.trial107:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-20T21:37:23.482 DEBUG:teuthology.orchestra.run.trial032:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:37:23.525 DEBUG:teuthology.orchestra.run.trial107:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:37:23.534 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:23.535 DEBUG:teuthology.orchestra.run.trial107:> dd if=/scratch_devs of=/dev/stdout 2026-02-20T21:37:23.579 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-20T21:37:23.579 DEBUG:teuthology.orchestra.run.trial107:> stat /dev/vg_nvme/lv_1 2026-02-20T21:37:23.626 INFO:teuthology.orchestra.run.trial107.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-20T21:37:23.627 INFO:teuthology.orchestra.run.trial107.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:37:23.627 INFO:teuthology.orchestra.run.trial107.stdout:Device: 5h/5d Inode: 899 Links: 1 2026-02-20T21:37:23.627 INFO:teuthology.orchestra.run.trial107.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:37:23.627 INFO:teuthology.orchestra.run.trial107.stdout:Access: 2026-02-20 21:36:10.911556786 +0000 2026-02-20T21:37:23.627 INFO:teuthology.orchestra.run.trial107.stdout:Modify: 2026-02-20 21:36:10.739559709 +0000 2026-02-20T21:37:23.627 INFO:teuthology.orchestra.run.trial107.stdout:Change: 2026-02-20 21:36:10.739559709 +0000 2026-02-20T21:37:23.627 INFO:teuthology.orchestra.run.trial107.stdout: Birth: - 2026-02-20T21:37:23.628 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-20T21:37:23.674 INFO:teuthology.orchestra.run.trial107.stderr:1+0 records in 2026-02-20T21:37:23.674 INFO:teuthology.orchestra.run.trial107.stderr:1+0 records out 2026-02-20T21:37:23.674 INFO:teuthology.orchestra.run.trial107.stderr:512 bytes copied, 0.000174124 s, 2.9 MB/s 2026-02-20T21:37:23.675 DEBUG:teuthology.orchestra.run.trial107:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-20T21:37:23.724 DEBUG:teuthology.orchestra.run.trial107:> stat /dev/vg_nvme/lv_2 2026-02-20T21:37:23.770 INFO:teuthology.orchestra.run.trial107.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-20T21:37:23.771 INFO:teuthology.orchestra.run.trial107.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:37:23.771 INFO:teuthology.orchestra.run.trial107.stdout:Device: 5h/5d Inode: 935 Links: 1 2026-02-20T21:37:23.771 INFO:teuthology.orchestra.run.trial107.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:37:23.771 INFO:teuthology.orchestra.run.trial107.stdout:Access: 2026-02-20 21:36:11.083553863 +0000 2026-02-20T21:37:23.771 INFO:teuthology.orchestra.run.trial107.stdout:Modify: 2026-02-20 21:36:11.079553929 +0000 2026-02-20T21:37:23.771 INFO:teuthology.orchestra.run.trial107.stdout:Change: 2026-02-20 21:36:11.079553929 +0000 2026-02-20T21:37:23.771 INFO:teuthology.orchestra.run.trial107.stdout: Birth: - 2026-02-20T21:37:23.772 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-20T21:37:23.818 INFO:teuthology.orchestra.run.trial107.stderr:1+0 records in 2026-02-20T21:37:23.818 INFO:teuthology.orchestra.run.trial107.stderr:1+0 records out 2026-02-20T21:37:23.818 INFO:teuthology.orchestra.run.trial107.stderr:512 bytes copied, 0.000160699 s, 3.2 MB/s 2026-02-20T21:37:23.819 DEBUG:teuthology.orchestra.run.trial107:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-20T21:37:23.868 DEBUG:teuthology.orchestra.run.trial107:> stat /dev/vg_nvme/lv_3 2026-02-20T21:37:23.915 INFO:teuthology.orchestra.run.trial107.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-20T21:37:23.915 INFO:teuthology.orchestra.run.trial107.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:37:23.915 INFO:teuthology.orchestra.run.trial107.stdout:Device: 5h/5d Inode: 947 Links: 1 2026-02-20T21:37:23.915 INFO:teuthology.orchestra.run.trial107.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:37:23.915 INFO:teuthology.orchestra.run.trial107.stdout:Access: 2026-02-20 21:36:11.619544751 +0000 2026-02-20T21:37:23.915 INFO:teuthology.orchestra.run.trial107.stdout:Modify: 2026-02-20 21:36:11.443547741 +0000 2026-02-20T21:37:23.915 INFO:teuthology.orchestra.run.trial107.stdout:Change: 2026-02-20 21:36:11.443547741 +0000 2026-02-20T21:37:23.915 INFO:teuthology.orchestra.run.trial107.stdout: Birth: - 2026-02-20T21:37:23.916 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-20T21:37:23.966 INFO:teuthology.orchestra.run.trial107.stderr:1+0 records in 2026-02-20T21:37:23.966 INFO:teuthology.orchestra.run.trial107.stderr:1+0 records out 2026-02-20T21:37:23.966 INFO:teuthology.orchestra.run.trial107.stderr:512 bytes copied, 0.000174385 s, 2.9 MB/s 2026-02-20T21:37:23.967 DEBUG:teuthology.orchestra.run.trial107:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-20T21:37:24.016 DEBUG:teuthology.orchestra.run.trial107:> stat /dev/vg_nvme/lv_4 2026-02-20T21:37:24.062 INFO:teuthology.orchestra.run.trial107.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-20T21:37:24.063 INFO:teuthology.orchestra.run.trial107.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:37:24.063 INFO:teuthology.orchestra.run.trial107.stdout:Device: 5h/5d Inode: 969 Links: 1 2026-02-20T21:37:24.063 INFO:teuthology.orchestra.run.trial107.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:37:24.063 INFO:teuthology.orchestra.run.trial107.stdout:Access: 2026-02-20 21:36:11.975538699 +0000 2026-02-20T21:37:24.063 INFO:teuthology.orchestra.run.trial107.stdout:Modify: 2026-02-20 21:36:11.795541759 +0000 2026-02-20T21:37:24.063 INFO:teuthology.orchestra.run.trial107.stdout:Change: 2026-02-20 21:36:11.795541759 +0000 2026-02-20T21:37:24.063 INFO:teuthology.orchestra.run.trial107.stdout: Birth: - 2026-02-20T21:37:24.064 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-20T21:37:24.110 INFO:teuthology.orchestra.run.trial107.stderr:1+0 records in 2026-02-20T21:37:24.110 INFO:teuthology.orchestra.run.trial107.stderr:1+0 records out 2026-02-20T21:37:24.110 INFO:teuthology.orchestra.run.trial107.stderr:512 bytes copied, 0.00017229 s, 3.0 MB/s 2026-02-20T21:37:24.111 DEBUG:teuthology.orchestra.run.trial107:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-20T21:37:24.160 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:37:24.161 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial107.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:37:24.161 INFO:tasks.ceph:Generating config... 2026-02-20T21:37:24.163 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-20T21:37:24.163 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-20T21:37:24.163 INFO:tasks.ceph:[client] log to stderr = False 2026-02-20T21:37:24.163 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-20T21:37:24.163 INFO:tasks.ceph:[client] rbd mirror image policy type = none 2026-02-20T21:37:24.163 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.163 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.164 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.164 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.164 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.164 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.164 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.164 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.164 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-20T21:37:24.164 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-20T21:37:24.164 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-20T21:37:24.164 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-20T21:37:24.164 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-20T21:37:24.165 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-20T21:37:24.165 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-20T21:37:24.165 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-20T21:37:24.165 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-20T21:37:24.165 INFO:tasks.ceph:[osd] bluestore compression algorithm = snappy 2026-02-20T21:37:24.165 INFO:tasks.ceph:[osd] bluestore compression mode = aggressive 2026-02-20T21:37:24.165 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-20T21:37:24.165 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-20T21:37:24.165 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-20T21:37:24.165 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-20T21:37:24.165 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-20T21:37:24.165 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-20T21:37:24.166 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-20T21:37:24.166 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-20T21:37:24.166 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-20T21:37:24.166 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-20T21:37:24.166 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-20T21:37:24.166 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-20T21:37:24.166 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-20T21:37:24.219 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster2.keyring 2026-02-20T21:37:24.221 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-20T21:37:24.282 DEBUG:teuthology.orchestra.run.trial107:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-20T21:37:24.330 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.107')] 2026-02-20T21:37:24.331 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.107', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluestore block size': 96636764160, 'bluestore compression algorithm': 'snappy', 'bluestore compression mode': 'aggressive', 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-20T21:37:24.332 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:24.333 DEBUG:teuthology.orchestra.run.trial107:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:37:24.386 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.107 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:37:24.457 INFO:teuthology.orchestra.run.trial107.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:37:24.457 INFO:teuthology.orchestra.run.trial107.stdout:monmaptool: generated fsid e67c6e85-7576-4986-9d91-ad2adc537194 2026-02-20T21:37:24.457 INFO:teuthology.orchestra.run.trial107.stdout:setting min_mon_release = tentacle 2026-02-20T21:37:24.458 INFO:teuthology.orchestra.run.trial107.stdout:epoch 0 2026-02-20T21:37:24.458 INFO:teuthology.orchestra.run.trial107.stdout:fsid e67c6e85-7576-4986-9d91-ad2adc537194 2026-02-20T21:37:24.458 INFO:teuthology.orchestra.run.trial107.stdout:last_changed 2026-02-20T21:37:24.454603+0000 2026-02-20T21:37:24.458 INFO:teuthology.orchestra.run.trial107.stdout:created 2026-02-20T21:37:24.454603+0000 2026-02-20T21:37:24.458 INFO:teuthology.orchestra.run.trial107.stdout:min_mon_release 20 (tentacle) 2026-02-20T21:37:24.458 INFO:teuthology.orchestra.run.trial107.stdout:election_strategy: 1 2026-02-20T21:37:24.458 INFO:teuthology.orchestra.run.trial107.stdout:0: [v2:10.20.193.107:3300/0,v1:10.20.193.107:6789/0] mon.a 2026-02-20T21:37:24.458 INFO:teuthology.orchestra.run.trial107.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-20T21:37:24.460 DEBUG:teuthology.orchestra.run.trial107:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:37:24.515 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID e67c6e85-7576-4986-9d91-ad2adc537194... 2026-02-20T21:37:24.518 DEBUG:teuthology.orchestra.run.trial032:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-20T21:37:24.522 DEBUG:teuthology.orchestra.run.trial107:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-20T21:37:24.563 INFO:teuthology.orchestra.run.trial032.stdout:[global] 2026-02-20T21:37:24.563 INFO:teuthology.orchestra.run.trial032.stdout: chdir = "" 2026-02-20T21:37:24.563 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: auth supported = cephx 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: filestore xattr use omap = true 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: mon clock drift allowed = 1.000 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: osd crush chooseleaf type = 0 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: auth debug = true 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.564 INFO:teuthology.orchestra.run.trial032.stdout: ms die on old message = true 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: ms die on bug = true 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: mon pg warn max object skew = 0 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: osd pool default size = 2 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: mon osd allow primary affinity = true 2026-02-20T21:37:24.565 INFO:teuthology.orchestra.run.trial032.stdout: mon osd allow pg remap = true 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on no sortbitwise = false 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on too few osds = false 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: mon_allow_pool_size_one = true 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:37:24.566 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: osd default data pool replay window = 5 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: mon allow pool delete = true 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: mon cluster log file level = debug 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: debug asserts on shutdown = true 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: mon health detail to clog = false 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: mon host = 10.20.193.107 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: mon client directed command retry = 5 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: mon warn on pool no app = False 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: ms inject socket failures = 5000 2026-02-20T21:37:24.567 INFO:teuthology.orchestra.run.trial032.stdout: fsid = e67c6e85-7576-4986-9d91-ad2adc537194 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout:[osd] 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: osd journal size = 100 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: osd scrub load threshold = 5.0 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: osd scrub max interval = 600 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: osd mclock skip benchmark = true 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: osd recover clone overlap = true 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: osd recovery max chunk = 1048576 2026-02-20T21:37:24.568 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: osd debug shutdown = true 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: osd debug op order = true 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: osd debug verify stray on activate = true 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: osd debug trim objects = true 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: osd open classes on start = true 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: osd debug pg log writeout = true 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.569 INFO:teuthology.orchestra.run.trial032.stdout: osd map max advance = 10 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: journal zero on create = true 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: filestore apply finisher threads = 3 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: bdev debug aio = true 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: osd debug misdirected ops = true 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: bluestore block size = 96636764160 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: bluestore compression algorithm = snappy 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: bluestore compression mode = aggressive 2026-02-20T21:37:24.570 INFO:teuthology.orchestra.run.trial032.stdout: bluestore fsck on mount = True 2026-02-20T21:37:24.571 INFO:teuthology.orchestra.run.trial032.stdout: debug bluefs = 1/20 2026-02-20T21:37:24.571 INFO:teuthology.orchestra.run.trial032.stdout: debug bluestore = 1/20 2026-02-20T21:37:24.571 INFO:teuthology.orchestra.run.trial032.stdout: debug ms = 1 2026-02-20T21:37:24.571 INFO:teuthology.orchestra.run.trial032.stdout: debug osd = 20 2026-02-20T21:37:24.571 INFO:teuthology.orchestra.run.trial032.stdout: debug rocksdb = 4/10 2026-02-20T21:37:24.571 INFO:teuthology.orchestra.run.trial032.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:37:24.571 INFO:teuthology.orchestra.run.trial032.stdout: mon osd full ratio = 0.9 2026-02-20T21:37:24.571 INFO:teuthology.orchestra.run.trial032.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:37:24.572 INFO:teuthology.orchestra.run.trial032.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:37:24.572 INFO:teuthology.orchestra.run.trial032.stdout: osd objectstore = bluestore 2026-02-20T21:37:24.572 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.572 INFO:teuthology.orchestra.run.trial032.stdout:[mgr] 2026-02-20T21:37:24.572 INFO:teuthology.orchestra.run.trial032.stdout: debug ms = 1 2026-02-20T21:37:24.572 INFO:teuthology.orchestra.run.trial032.stdout: debug mgr = 20 2026-02-20T21:37:24.572 INFO:teuthology.orchestra.run.trial032.stdout: debug mon = 20 2026-02-20T21:37:24.572 INFO:teuthology.orchestra.run.trial032.stdout: debug auth = 20 2026-02-20T21:37:24.572 INFO:teuthology.orchestra.run.trial032.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: mgr/telemetry/nag = false 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout:[mon] 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: debug ms = 1 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: debug mon = 20 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: debug paxos = 20 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: debug auth = 20 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: mon data avail warn = 5 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: mon mgr mkfs grace = 240 2026-02-20T21:37:24.573 INFO:teuthology.orchestra.run.trial032.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: mon osd reporter subtree level = osd 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: mon osd prime pg temp = true 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:37:24.574 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:37:24.575 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.575 INFO:teuthology.orchestra.run.trial032.stdout: # 1m isn't quite enough 2026-02-20T21:37:24.575 INFO:teuthology.orchestra.run.trial032.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:37:24.575 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.575 INFO:teuthology.orchestra.run.trial032.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:37:24.575 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T21:37:24.575 INFO:teuthology.orchestra.run.trial032.stdout:[client] 2026-02-20T21:37:24.575 INFO:teuthology.orchestra.run.trial032.stdout: rgw cache enabled = true 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout: rgw enable ops log = true 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout: rgw enable usage log = true 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout: debug rbd = 20 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout: debug rbd_mirror = 15 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout: log to stderr = False 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout: rbd default features = 125 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout: rbd mirror image policy type = none 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout:[mon.a] 2026-02-20T21:37:24.576 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.0] 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.1] 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.2] 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout:[client.mirror.3] 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.577 INFO:teuthology.orchestra.run.trial032.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.582 INFO:teuthology.orchestra.run.trial107.stdout:[global] 2026-02-20T21:37:24.582 INFO:teuthology.orchestra.run.trial107.stdout: chdir = "" 2026-02-20T21:37:24.582 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:37:24.582 INFO:teuthology.orchestra.run.trial107.stdout: auth supported = cephx 2026-02-20T21:37:24.582 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.582 INFO:teuthology.orchestra.run.trial107.stdout: filestore xattr use omap = true 2026-02-20T21:37:24.583 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.583 INFO:teuthology.orchestra.run.trial107.stdout: mon clock drift allowed = 1.000 2026-02-20T21:37:24.583 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.583 INFO:teuthology.orchestra.run.trial107.stdout: osd crush chooseleaf type = 0 2026-02-20T21:37:24.583 INFO:teuthology.orchestra.run.trial107.stdout: auth debug = true 2026-02-20T21:37:24.583 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.583 INFO:teuthology.orchestra.run.trial107.stdout: ms die on old message = true 2026-02-20T21:37:24.583 INFO:teuthology.orchestra.run.trial107.stdout: ms die on bug = true 2026-02-20T21:37:24.583 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: mon pg warn max object skew = 0 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: osd pool default size = 2 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: mon osd allow primary affinity = true 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: mon osd allow pg remap = true 2026-02-20T21:37:24.584 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on no sortbitwise = false 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on too few osds = false 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: mon_allow_pool_size_one = true 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: osd default data pool replay window = 5 2026-02-20T21:37:24.585 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: mon allow pool delete = true 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: mon cluster log file level = debug 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: debug asserts on shutdown = true 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: mon health detail to clog = false 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: mon host = 10.20.193.107 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: mon client directed command retry = 5 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: mon warn on pool no app = False 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: ms inject socket failures = 5000 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: fsid = e67c6e85-7576-4986-9d91-ad2adc537194 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.586 INFO:teuthology.orchestra.run.trial107.stdout:[osd] 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: osd journal size = 100 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: osd scrub load threshold = 5.0 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: osd scrub max interval = 600 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: osd mclock skip benchmark = true 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: osd recover clone overlap = true 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: osd recovery max chunk = 1048576 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: osd debug shutdown = true 2026-02-20T21:37:24.587 INFO:teuthology.orchestra.run.trial107.stdout: osd debug op order = true 2026-02-20T21:37:24.588 INFO:teuthology.orchestra.run.trial107.stdout: osd debug verify stray on activate = true 2026-02-20T21:37:24.588 INFO:teuthology.orchestra.run.trial107.stdout: osd debug trim objects = true 2026-02-20T21:37:24.588 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.588 INFO:teuthology.orchestra.run.trial107.stdout: osd open classes on start = true 2026-02-20T21:37:24.588 INFO:teuthology.orchestra.run.trial107.stdout: osd debug pg log writeout = true 2026-02-20T21:37:24.588 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.588 INFO:teuthology.orchestra.run.trial107.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:37:24.588 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.589 INFO:teuthology.orchestra.run.trial107.stdout: osd map max advance = 10 2026-02-20T21:37:24.589 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.589 INFO:teuthology.orchestra.run.trial107.stdout: journal zero on create = true 2026-02-20T21:37:24.589 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.589 INFO:teuthology.orchestra.run.trial107.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:37:24.589 INFO:teuthology.orchestra.run.trial107.stdout: filestore apply finisher threads = 3 2026-02-20T21:37:24.589 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.589 INFO:teuthology.orchestra.run.trial107.stdout: bdev debug aio = true 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: osd debug misdirected ops = true 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: bluestore block size = 96636764160 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: bluestore compression algorithm = snappy 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: bluestore compression mode = aggressive 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: bluestore fsck on mount = True 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: debug bluefs = 1/20 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: debug bluestore = 1/20 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: debug ms = 1 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: debug osd = 20 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: debug rocksdb = 4/10 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:37:24.590 INFO:teuthology.orchestra.run.trial107.stdout: mon osd full ratio = 0.9 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: osd objectstore = bluestore 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout:[mgr] 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: debug ms = 1 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: debug mgr = 20 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: debug mon = 20 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: debug auth = 20 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:37:24.591 INFO:teuthology.orchestra.run.trial107.stdout: mgr/telemetry/nag = false 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout:[mon] 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: debug ms = 1 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: debug mon = 20 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: debug paxos = 20 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: debug auth = 20 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: mon data avail warn = 5 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: mon mgr mkfs grace = 240 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: mon osd reporter subtree level = osd 2026-02-20T21:37:24.592 INFO:teuthology.orchestra.run.trial107.stdout: mon osd prime pg temp = true 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: # 1m isn't quite enough 2026-02-20T21:37:24.593 INFO:teuthology.orchestra.run.trial107.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout:[client] 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: rgw cache enabled = true 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: rgw enable ops log = true 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: rgw enable usage log = true 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: debug rbd = 20 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: debug rbd_mirror = 15 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: log to stderr = False 2026-02-20T21:37:24.594 INFO:teuthology.orchestra.run.trial107.stdout: rbd default features = 125 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout: rbd mirror image policy type = none 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout:[mon.a] 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.0] 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.1] 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.2] 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.3] 2026-02-20T21:37:24.595 INFO:teuthology.orchestra.run.trial107.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:37:24.596 INFO:teuthology.orchestra.run.trial107.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:37:24.597 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-20T21:37:24.597 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-20T21:37:24.662 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-20T21:37:24.663 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:24.663 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-20T21:37:24.711 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:24.711 DEBUG:teuthology.orchestra.run.trial107:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-20T21:37:24.759 INFO:tasks.ceph:Sending monmap to node ubuntu@trial032.front.sepia.ceph.com 2026-02-20T21:37:24.759 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:24.759 DEBUG:teuthology.orchestra.run.trial032:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-20T21:37:24.760 DEBUG:teuthology.orchestra.run.trial032:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-20T21:37:24.774 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:24.774 DEBUG:teuthology.orchestra.run.trial032:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:37:24.820 INFO:tasks.ceph:Sending monmap to node ubuntu@trial107.front.sepia.ceph.com 2026-02-20T21:37:24.821 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:24.821 DEBUG:teuthology.orchestra.run.trial107:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-20T21:37:24.821 DEBUG:teuthology.orchestra.run.trial107:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-20T21:37:24.836 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:24.836 DEBUG:teuthology.orchestra.run.trial107:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:37:24.887 INFO:tasks.ceph:Setting up mon nodes... 2026-02-20T21:37:24.887 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-20T21:37:24.887 DEBUG:teuthology.orchestra.run.trial032:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-20T21:37:24.911 INFO:teuthology.orchestra.run.trial032.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-20T21:37:24.914 INFO:tasks.ceph:Setting up mds nodes... 2026-02-20T21:37:24.914 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-20T21:37:24.914 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-20T21:37:24.973 INFO:teuthology.orchestra.run.trial032.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-20T21:37:24.980 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-20T21:37:25.006 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-20T21:37:25.014 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-20T21:37:25.082 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-20T21:37:25.090 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-20T21:37:25.157 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-20T21:37:25.166 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-20T21:37:25.233 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-20T21:37:25.241 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-20T21:37:25.303 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-20T21:37:25.310 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-20T21:37:25.373 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-20T21:37:25.381 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-20T21:37:25.445 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-20T21:37:25.453 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-20T21:37:25.511 INFO:teuthology.orchestra.run.trial107.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-20T21:37:25.518 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-20T21:37:25.519 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial032.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial107.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:37:25.519 DEBUG:teuthology.orchestra.run.trial107:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-20T21:37:25.567 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:37:25.567 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-20T21:37:25.567 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial107.front.sepia.ceph.com 2026-02-20T21:37:25.567 DEBUG:teuthology.orchestra.run.trial107:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-20T21:37:25.617 INFO:teuthology.orchestra.run.trial107.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:37:25.617 INFO:teuthology.orchestra.run.trial107.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:37:25.617 INFO:teuthology.orchestra.run.trial107.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:37:25.617 INFO:teuthology.orchestra.run.trial107.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-20T21:37:25.617 INFO:teuthology.orchestra.run.trial107.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:37:25.617 INFO:teuthology.orchestra.run.trial107.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:37:25.617 INFO:teuthology.orchestra.run.trial107.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:37:25.617 INFO:teuthology.orchestra.run.trial107.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:37:25.617 INFO:teuthology.orchestra.run.trial107.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:37:25.618 INFO:teuthology.orchestra.run.trial107.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:37:26.193 INFO:teuthology.orchestra.run.trial107.stdout:Discarding blocks...Done. 2026-02-20T21:37:26.195 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial107.front.sepia.ceph.com -o noatime 2026-02-20T21:37:26.195 DEBUG:teuthology.orchestra.run.trial107:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-20T21:37:26.248 DEBUG:teuthology.orchestra.run.trial107:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-20T21:37:26.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:37:26.301 INFO:teuthology.orchestra.run.trial107.stderr:sudo: /sbin/restorecon: command not found 2026-02-20T21:37:26.302 DEBUG:teuthology.orchestra.run.trial107:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-20T21:37:26.351 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:37:26.351 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-20T21:37:26.351 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial107.front.sepia.ceph.com 2026-02-20T21:37:26.352 DEBUG:teuthology.orchestra.run.trial107:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-20T21:37:26.409 INFO:teuthology.orchestra.run.trial107.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:37:26.409 INFO:teuthology.orchestra.run.trial107.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:37:26.409 INFO:teuthology.orchestra.run.trial107.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:37:26.409 INFO:teuthology.orchestra.run.trial107.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-20T21:37:26.409 INFO:teuthology.orchestra.run.trial107.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:37:26.410 INFO:teuthology.orchestra.run.trial107.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:37:26.410 INFO:teuthology.orchestra.run.trial107.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:37:26.410 INFO:teuthology.orchestra.run.trial107.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:37:26.410 INFO:teuthology.orchestra.run.trial107.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:37:26.410 INFO:teuthology.orchestra.run.trial107.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:37:26.974 INFO:teuthology.orchestra.run.trial107.stdout:Discarding blocks...Done. 2026-02-20T21:37:26.976 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial107.front.sepia.ceph.com -o noatime 2026-02-20T21:37:26.976 DEBUG:teuthology.orchestra.run.trial107:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-20T21:37:27.036 DEBUG:teuthology.orchestra.run.trial107:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-20T21:37:27.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:37:27.085 INFO:teuthology.orchestra.run.trial107.stderr:sudo: /sbin/restorecon: command not found 2026-02-20T21:37:27.086 DEBUG:teuthology.orchestra.run.trial107:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-20T21:37:27.139 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:37:27.139 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-20T21:37:27.139 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial107.front.sepia.ceph.com 2026-02-20T21:37:27.140 DEBUG:teuthology.orchestra.run.trial107:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-20T21:37:27.192 INFO:teuthology.orchestra.run.trial107.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:37:27.192 INFO:teuthology.orchestra.run.trial107.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:37:27.193 INFO:teuthology.orchestra.run.trial107.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:37:27.193 INFO:teuthology.orchestra.run.trial107.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-20T21:37:27.193 INFO:teuthology.orchestra.run.trial107.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:37:27.193 INFO:teuthology.orchestra.run.trial107.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:37:27.193 INFO:teuthology.orchestra.run.trial107.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:37:27.193 INFO:teuthology.orchestra.run.trial107.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:37:27.193 INFO:teuthology.orchestra.run.trial107.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:37:27.193 INFO:teuthology.orchestra.run.trial107.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:37:27.755 INFO:teuthology.orchestra.run.trial107.stdout:Discarding blocks...Done. 2026-02-20T21:37:27.756 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial107.front.sepia.ceph.com -o noatime 2026-02-20T21:37:27.757 DEBUG:teuthology.orchestra.run.trial107:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-20T21:37:27.816 DEBUG:teuthology.orchestra.run.trial107:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-20T21:37:27.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:37:27.869 INFO:teuthology.orchestra.run.trial107.stderr:sudo: /sbin/restorecon: command not found 2026-02-20T21:37:27.869 DEBUG:teuthology.orchestra.run.trial107:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:37:27.932 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:27.927+0000 7fee574daa40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-20T21:37:27.933 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:27.927+0000 7fee574daa40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-20T21:37:27.933 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:27.927+0000 7fee574daa40 -1 bdev(0x55a5885b9800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-20T21:37:27.933 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:27.927+0000 7fee574daa40 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-20T21:37:28.849 DEBUG:teuthology.orchestra.run.trial107:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-20T21:37:28.899 DEBUG:teuthology.orchestra.run.trial107:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:37:28.964 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:28.959+0000 7f0d29580a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-20T21:37:28.964 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:28.959+0000 7f0d29580a40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-20T21:37:28.964 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:28.959+0000 7f0d29580a40 -1 bdev(0x555e9f94d800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-20T21:37:28.964 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:28.959+0000 7f0d29580a40 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-20T21:37:29.792 DEBUG:teuthology.orchestra.run.trial107:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-20T21:37:29.843 DEBUG:teuthology.orchestra.run.trial107:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:37:29.907 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:29.903+0000 7f429bfe5a40 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-20T21:37:29.907 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:29.903+0000 7f429bfe5a40 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-20T21:37:29.907 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:29.903+0000 7f429bfe5a40 -1 bdev(0x55a310821800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-20T21:37:29.907 INFO:teuthology.orchestra.run.trial107.stderr:2026-02-20T21:37:29.903+0000 7f429bfe5a40 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-20T21:37:30.792 DEBUG:teuthology.orchestra.run.trial107:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-20T21:37:30.844 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-20T21:37:30.844 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:30.844 DEBUG:teuthology.orchestra.run.trial032:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-20T21:37:30.855 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:30.855 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-20T21:37:30.896 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:30.896 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-20T21:37:30.948 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:30.948 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-20T21:37:31.000 DEBUG:teuthology.orchestra.run.trial032:> set -ex 2026-02-20T21:37:31.000 DEBUG:teuthology.orchestra.run.trial032:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-20T21:37:31.005 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:31.005 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-20T21:37:31.047 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:31.047 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-20T21:37:31.095 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:31.095 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-20T21:37:31.143 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:31.143 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-20T21:37:31.187 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:31.187 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-20T21:37:31.231 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:31.231 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-20T21:37:31.279 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:31.280 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-20T21:37:31.327 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:31.327 DEBUG:teuthology.orchestra.run.trial107:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-20T21:37:31.375 INFO:tasks.ceph:Adding keys to all mons... 2026-02-20T21:37:31.375 DEBUG:teuthology.orchestra.run.trial107:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-20T21:37:31.425 INFO:teuthology.orchestra.run.trial107.stdout:[mgr.x] 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAU1JhpzIZhNhAAPmesHHA2LSJplQbF2L8qig== 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout:[osd.0] 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAX1Jhp0FxuNxAAq9mvS+Vf8zQpEJn/jdtEuw== 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout:[osd.1] 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAY1JhpkypUORAAIKbZ1SZ+vq4E7ZhZQkaF6g== 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout:[osd.2] 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAZ1JhpoOrzNRAA5qED8MtXbPWqbBpsZb7Psg== 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout:[client.0] 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAU1Jhp65kROhAA8LRizhuDVUrj6yyfoaN2hg== 2026-02-20T21:37:31.426 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror] 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpUEs5ABAA48cQEWIakhFsZFRz1Prubg== 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.0] 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpXgvABBAANWJSRLNVXHhmcRYZ6ATBPw== 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.1] 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpBj47CRAAbcMmIFAoKgmDR/7tXTM9JQ== 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.2] 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpX+vADRAAjjpWfVvdIqznng+Gx8rWbg== 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.3] 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpBAT2ERAA/QNn2QUxZhUummtv4ob0/Q== 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.4] 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpQAQeFhAAW0Sfs09PF8TX8KyRfU6wLg== 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.5] 2026-02-20T21:37:31.427 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpW0plGhAAxXGF5hal0ukzCTmEh4rZ7Q== 2026-02-20T21:37:31.428 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.6] 2026-02-20T21:37:31.428 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1Jhp1XBbHhAAccsNSCU9/GGryT3rUU2egQ== 2026-02-20T21:37:31.429 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-20T21:37:31.493 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:37:31.557 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:37:31.621 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:37:31.684 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:31.749 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:31.813 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:31.877 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:31.940 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:32.004 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:32.069 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:32.132 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:32.196 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:37:32.260 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-20T21:37:32.261 DEBUG:teuthology.orchestra.run.trial107:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-20T21:37:32.311 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-20T21:37:32.381 DEBUG:teuthology.orchestra.run.trial107:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-20T21:37:32.431 DEBUG:teuthology.orchestra.run.trial107:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:37:32.478 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-20T21:37:32.479 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-20T21:37:32.479 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-20T21:37:32.520 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-20T21:37:32.520 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-20T21:37:32.521 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-20T21:37:32.521 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-20T21:37:32.523 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-20T21:37:32.523 DEBUG:tasks.ceph:set 0 configs 2026-02-20T21:37:32.524 DEBUG:teuthology.orchestra.run.trial107:> sudo ceph --cluster cluster2 config dump 2026-02-20T21:37:32.642 INFO:teuthology.orchestra.run.trial107.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-20T21:37:32.652 INFO:tasks.ceph:Setting crush tunables to default 2026-02-20T21:37:32.652 DEBUG:teuthology.orchestra.run.trial107:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-20T21:37:32.749 INFO:teuthology.orchestra.run.trial107.stderr:adjusted tunables profile to default 2026-02-20T21:37:32.758 INFO:tasks.ceph:check_enable_crimson: False 2026-02-20T21:37:32.758 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-20T21:37:32.759 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:32.759 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-20T21:37:32.768 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:32.768 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-20T21:37:32.820 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:37:32.820 DEBUG:teuthology.orchestra.run.trial107:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-20T21:37:32.871 DEBUG:teuthology.orchestra.run.trial107:> sudo ceph --cluster cluster2 osd new 418d8210-8c01-45cb-abd1-b38a88311d87 0 2026-02-20T21:37:33.008 INFO:teuthology.orchestra.run.trial107.stdout:0 2026-02-20T21:37:33.017 DEBUG:teuthology.orchestra.run.trial107:> sudo ceph --cluster cluster2 osd new 75cc25e0-d127-4511-8dfa-438ba34f62c6 1 2026-02-20T21:37:33.113 INFO:teuthology.orchestra.run.trial107.stdout:1 2026-02-20T21:37:33.125 DEBUG:teuthology.orchestra.run.trial107:> sudo ceph --cluster cluster2 osd new d25ad5e6-49c5-486e-b22d-e4a8966305d8 2 2026-02-20T21:37:33.222 INFO:teuthology.orchestra.run.trial107.stdout:2 2026-02-20T21:37:33.232 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-20T21:37:33.232 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-20T21:37:33.234 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-20T21:37:33.234 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-20T21:37:33.234 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-20T21:37:33.236 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-20T21:37:33.236 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-20T21:37:33.237 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-20T21:37:33.238 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-20T21:37:33.239 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:37:33.336 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:33.336 INFO:teuthology.orchestra.run.trial107.stdout:{"epoch":5,"fsid":"e67c6e85-7576-4986-9d91-ad2adc537194","created":"2026-02-20T21:37:32.598295+0000","modified":"2026-02-20T21:37:33.219748+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:37:33.219748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"418d8210-8c01-45cb-abd1-b38a88311d87","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"75cc25e0-d127-4511-8dfa-438ba34f62c6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"d25ad5e6-49c5-486e-b22d-e4a8966305d8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:33.346 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-20T21:37:33.347 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-20T21:37:33.649 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:37:33.732 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:37:33.727+0000 7fa5f37bca40 -1 Falling back to public interface 2026-02-20T21:37:33.748 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:37:33.743+0000 7f255f77ca40 -1 Falling back to public interface 2026-02-20T21:37:33.768 INFO:tasks.ceph.cluster2.osd.1.trial107.stderr:2026-02-20T21:37:33.763+0000 7f6bad6ffa40 -1 Falling back to public interface 2026-02-20T21:37:33.789 INFO:tasks.ceph.cluster2.mgr.x.trial032.stderr:ERROR:root:Module 'jmespath' is not installed. 2026-02-20T21:37:33.827 INFO:tasks.ceph.cluster2.mgr.x.trial032.stderr:ERROR:dashboard.controllers.nvmeof:Failed to import NVMeoFClient and related components: cannot import name 'NVMeoFClient' from 'dashboard.services.nvmeof_client' (/usr/share/ceph/mgr/dashboard/services/nvmeof_client.py) 2026-02-20T21:37:33.828 INFO:tasks.ceph.cluster2.mgr.x.trial032.stderr:ERROR:sso:Module 'jmespath' is not installed. 2026-02-20T21:37:33.842 INFO:tasks.ceph.cluster2.mgr.x.trial032.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-20T21:37:33.908 INFO:teuthology.misc.health.trial107.stderr:2026-02-20T21:37:33.903+0000 7fa9eaffd640 0 --2- 10.20.193.107:0/2567499495 >> v2:10.20.193.107:3300/0 conn(0x7fa9ec09c640 0x7fa9ec158ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-20T21:37:33.941 INFO:teuthology.misc.health.trial107.stdout: 2026-02-20T21:37:33.941 INFO:teuthology.misc.health.trial107.stdout:{"epoch":5,"fsid":"e67c6e85-7576-4986-9d91-ad2adc537194","created":"2026-02-20T21:37:32.598295+0000","modified":"2026-02-20T21:37:33.219748+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:37:33.219748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"418d8210-8c01-45cb-abd1-b38a88311d87","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"75cc25e0-d127-4511-8dfa-438ba34f62c6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"d25ad5e6-49c5-486e-b22d-e4a8966305d8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:33.949 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-20T21:37:34.198 INFO:tasks.ceph.cluster2.mgr.x.trial032.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-20T21:37:34.199 INFO:tasks.ceph.cluster2.mgr.x.trial032.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-20T21:37:34.199 INFO:tasks.ceph.cluster2.mgr.x.trial032.stderr: from numpy import show_config as show_numpy_config 2026-02-20T21:37:34.851 INFO:tasks.ceph.cluster2.osd.1.trial107.stderr:2026-02-20T21:37:34.843+0000 7f6bad6ffa40 -1 osd.1 0 log_to_monitors true 2026-02-20T21:37:34.853 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:37:34.847+0000 7fa5f37bca40 -1 osd.0 0 log_to_monitors true 2026-02-20T21:37:34.892 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:37:34.887+0000 7f255f77ca40 -1 osd.2 0 log_to_monitors true 2026-02-20T21:37:35.461 INFO:tasks.ceph.cluster2.mgr.x.trial032.stderr:2026-02-20T21:37:35.458+0000 7fd689ba1640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-20T21:37:36.631 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:37:36.627+0000 7fa5ef765640 -1 osd.0 0 waiting for initial osdmap 2026-02-20T21:37:36.632 INFO:tasks.ceph.cluster2.osd.1.trial107.stderr:2026-02-20T21:37:36.627+0000 7f6ba96a8640 -1 osd.1 0 waiting for initial osdmap 2026-02-20T21:37:36.632 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:37:36.627+0000 7f255b725640 -1 osd.2 0 waiting for initial osdmap 2026-02-20T21:37:36.634 INFO:tasks.ceph.cluster2.osd.1.trial107.stderr:2026-02-20T21:37:36.627+0000 7f6ba0caf640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:37:36.634 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:37:36.627+0000 7fa5e6d6c640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:37:36.634 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:37:36.627+0000 7f2552d2c640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:37:40.253 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:37:40.400 INFO:teuthology.misc.health.trial107.stdout: 2026-02-20T21:37:40.401 INFO:teuthology.misc.health.trial107.stdout:{"epoch":10,"fsid":"e67c6e85-7576-4986-9d91-ad2adc537194","created":"2026-02-20T21:37:32.598295+0000","modified":"2026-02-20T21:37:39.614348+0000","last_up_change":"2026-02-20T21:37:37.608773+0000","last_in_change":"2026-02-20T21:37:33.219748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:37:38.286097+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"10","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"418d8210-8c01-45cb-abd1-b38a88311d87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6800","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6801","nonce":4201493740}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6802","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6803","nonce":4201493740}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6806","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6807","nonce":4201493740}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6804","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6805","nonce":4201493740}]},"public_addr":"10.20.193.107:6801/4201493740","cluster_addr":"10.20.193.107:6803/4201493740","heartbeat_back_addr":"10.20.193.107:6807/4201493740","heartbeat_front_addr":"10.20.193.107:6805/4201493740","state":["exists","up"]},{"osd":1,"uuid":"75cc25e0-d127-4511-8dfa-438ba34f62c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6816","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6817","nonce":1493137965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6818","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6819","nonce":1493137965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6822","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6823","nonce":1493137965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6820","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6821","nonce":1493137965}]},"public_addr":"10.20.193.107:6817/1493137965","cluster_addr":"10.20.193.107:6819/1493137965","heartbeat_back_addr":"10.20.193.107:6823/1493137965","heartbeat_front_addr":"10.20.193.107:6821/1493137965","state":["exists","up"]},{"osd":2,"uuid":"d25ad5e6-49c5-486e-b22d-e4a8966305d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6808","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6809","nonce":884104108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6810","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6811","nonce":884104108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6814","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6815","nonce":884104108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6812","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6813","nonce":884104108}]},"public_addr":"10.20.193.107:6809/884104108","cluster_addr":"10.20.193.107:6811/884104108","heartbeat_back_addr":"10.20.193.107:6815/884104108","heartbeat_front_addr":"10.20.193.107:6813/884104108","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:40.410 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-20T21:37:40.411 INFO:tasks.ceph:Creating RBD pool 2026-02-20T21:37:40.411 DEBUG:teuthology.orchestra.run.trial107:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-20T21:37:40.619 INFO:teuthology.orchestra.run.trial107.stderr:pool 'rbd' created 2026-02-20T21:37:40.632 DEBUG:teuthology.orchestra.run.trial107:> rbd --cluster cluster2 pool init rbd 2026-02-20T21:37:43.636 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-20T21:37:43.637 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2026-02-20T21:37:43.638 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-20T21:37:43.796 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:43.806 INFO:teuthology.orchestra.run.trial107.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-20T21:37:43.807 INFO:tasks.ceph_manager:config epoch is 1 2026-02-20T21:37:43.807 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-20T21:37:43.807 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-20T21:37:43.807 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-20T21:37:43.980 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:43.991 INFO:teuthology.orchestra.run.trial107.stdout:{"epoch":5,"flags":0,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6826","nonce":4196237434},{"type":"v1","addr":"10.20.193.32:6827","nonce":4196237434}]},"active_addr":"10.20.193.32:6827/4196237434","active_change":"2026-02-20T21:37:34.249013+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.32:0","nonce":2520339138}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.32:0","nonce":1091044279}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.32:0","nonce":3180307081}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.32:0","nonce":741573111}]}]} 2026-02-20T21:37:43.999 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-20T21:37:43.999 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-20T21:37:43.999 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:37:44.159 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:44.159 INFO:teuthology.orchestra.run.trial107.stdout:{"epoch":14,"fsid":"e67c6e85-7576-4986-9d91-ad2adc537194","created":"2026-02-20T21:37:32.598295+0000","modified":"2026-02-20T21:37:43.623434+0000","last_up_change":"2026-02-20T21:37:37.608773+0000","last_in_change":"2026-02-20T21:37:33.219748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:37:38.286097+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:37:40.560120+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"418d8210-8c01-45cb-abd1-b38a88311d87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6800","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6801","nonce":4201493740}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6802","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6803","nonce":4201493740}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6806","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6807","nonce":4201493740}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6804","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6805","nonce":4201493740}]},"public_addr":"10.20.193.107:6801/4201493740","cluster_addr":"10.20.193.107:6803/4201493740","heartbeat_back_addr":"10.20.193.107:6807/4201493740","heartbeat_front_addr":"10.20.193.107:6805/4201493740","state":["exists","up"]},{"osd":1,"uuid":"75cc25e0-d127-4511-8dfa-438ba34f62c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6816","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6817","nonce":1493137965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6818","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6819","nonce":1493137965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6822","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6823","nonce":1493137965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6820","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6821","nonce":1493137965}]},"public_addr":"10.20.193.107:6817/1493137965","cluster_addr":"10.20.193.107:6819/1493137965","heartbeat_back_addr":"10.20.193.107:6823/1493137965","heartbeat_front_addr":"10.20.193.107:6821/1493137965","state":["exists","up"]},{"osd":2,"uuid":"d25ad5e6-49c5-486e-b22d-e4a8966305d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6808","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6809","nonce":884104108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6810","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6811","nonce":884104108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6814","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6815","nonce":884104108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6812","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6813","nonce":884104108}]},"public_addr":"10.20.193.107:6809/884104108","cluster_addr":"10.20.193.107:6811/884104108","heartbeat_back_addr":"10.20.193.107:6815/884104108","heartbeat_front_addr":"10.20.193.107:6813/884104108","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:44.170 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-20T21:37:44.170 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:37:44.335 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:44.335 INFO:teuthology.orchestra.run.trial107.stdout:{"epoch":14,"fsid":"e67c6e85-7576-4986-9d91-ad2adc537194","created":"2026-02-20T21:37:32.598295+0000","modified":"2026-02-20T21:37:43.623434+0000","last_up_change":"2026-02-20T21:37:37.608773+0000","last_in_change":"2026-02-20T21:37:33.219748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:37:38.286097+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:37:40.560120+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"418d8210-8c01-45cb-abd1-b38a88311d87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6800","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6801","nonce":4201493740}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6802","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6803","nonce":4201493740}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6806","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6807","nonce":4201493740}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6804","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6805","nonce":4201493740}]},"public_addr":"10.20.193.107:6801/4201493740","cluster_addr":"10.20.193.107:6803/4201493740","heartbeat_back_addr":"10.20.193.107:6807/4201493740","heartbeat_front_addr":"10.20.193.107:6805/4201493740","state":["exists","up"]},{"osd":1,"uuid":"75cc25e0-d127-4511-8dfa-438ba34f62c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6816","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6817","nonce":1493137965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6818","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6819","nonce":1493137965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6822","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6823","nonce":1493137965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6820","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6821","nonce":1493137965}]},"public_addr":"10.20.193.107:6817/1493137965","cluster_addr":"10.20.193.107:6819/1493137965","heartbeat_back_addr":"10.20.193.107:6823/1493137965","heartbeat_front_addr":"10.20.193.107:6821/1493137965","state":["exists","up"]},{"osd":2,"uuid":"d25ad5e6-49c5-486e-b22d-e4a8966305d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6808","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6809","nonce":884104108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6810","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6811","nonce":884104108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6814","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6815","nonce":884104108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6812","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6813","nonce":884104108}]},"public_addr":"10.20.193.107:6809/884104108","cluster_addr":"10.20.193.107:6811/884104108","heartbeat_back_addr":"10.20.193.107:6815/884104108","heartbeat_front_addr":"10.20.193.107:6813/884104108","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:37:44.346 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-20T21:37:44.347 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-20T21:37:44.347 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-20T21:37:44.437 INFO:teuthology.orchestra.run.trial107.stdout:34359738372 2026-02-20T21:37:44.437 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-20T21:37:44.443 INFO:teuthology.orchestra.run.trial107.stdout:34359738372 2026-02-20T21:37:44.445 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-20T21:37:44.447 INFO:teuthology.orchestra.run.trial107.stdout:34359738371 2026-02-20T21:37:44.447 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-20T21:37:44.601 INFO:teuthology.orchestra.run.trial107.stdout:34359738370 2026-02-20T21:37:44.613 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738370 for osd.2 2026-02-20T21:37:44.637 INFO:teuthology.orchestra.run.trial107.stdout:34359738370 2026-02-20T21:37:44.644 INFO:teuthology.orchestra.run.trial107.stdout:34359738371 2026-02-20T21:37:44.650 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.1 2026-02-20T21:37:44.656 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-20T21:37:45.614 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-20T21:37:45.650 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-20T21:37:45.657 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-20T21:37:45.782 INFO:teuthology.orchestra.run.trial107.stdout:34359738371 2026-02-20T21:37:45.793 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.2 2026-02-20T21:37:45.805 INFO:teuthology.orchestra.run.trial107.stdout:34359738370 2026-02-20T21:37:45.812 INFO:teuthology.orchestra.run.trial107.stdout:34359738371 2026-02-20T21:37:45.817 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.1 2026-02-20T21:37:45.824 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-20T21:37:46.794 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-20T21:37:46.818 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-20T21:37:46.825 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-20T21:37:46.948 INFO:teuthology.orchestra.run.trial107.stdout:34359738372 2026-02-20T21:37:46.959 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.2 2026-02-20T21:37:46.960 DEBUG:teuthology.parallel:result is None 2026-02-20T21:37:46.987 INFO:teuthology.orchestra.run.trial107.stdout:34359738372 2026-02-20T21:37:46.991 INFO:teuthology.orchestra.run.trial107.stdout:34359738371 2026-02-20T21:37:46.999 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.0 2026-02-20T21:37:47.000 DEBUG:teuthology.parallel:result is None 2026-02-20T21:37:47.003 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738371 for osd.1 2026-02-20T21:37:47.003 DEBUG:teuthology.parallel:result is None 2026-02-20T21:37:47.003 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-20T21:37:47.003 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:37:47.200 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:47.200 INFO:teuthology.orchestra.run.trial107.stderr:dumped all 2026-02-20T21:37:47.212 INFO:teuthology.orchestra.run.trial107.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-20T21:37:46.256763+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81376,"kb_used_data":496,"kb_used_omap":19,"kb_used_meta":80812,"kb_avail":283034144,"statfs":{"total":289910292480,"available":289826963456,"internally_reserved":0,"allocated":507904,"data_stored":1287022,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":19825,"internal_metadata":82752143},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.630683"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.629154+0000","last_change":"2026-02-20T21:37:43.629317+0000","last_active":"2026-02-20T21:37:43.629154+0000","last_peered":"2026-02-20T21:37:43.629154+0000","last_clean":"2026-02-20T21:37:43.629154+0000","last_became_active":"2026-02-20T21:37:41.626252+0000","last_became_peered":"2026-02-20T21:37:41.626252+0000","last_unstale":"2026-02-20T21:37:43.629154+0000","last_undegraded":"2026-02-20T21:37:43.629154+0000","last_fullsized":"2026-02-20T21:37:43.629154+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:57:26.704591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000878194,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628725+0000","last_change":"2026-02-20T21:37:43.628857+0000","last_active":"2026-02-20T21:37:43.628725+0000","last_peered":"2026-02-20T21:37:43.628725+0000","last_clean":"2026-02-20T21:37:43.628725+0000","last_became_active":"2026-02-20T21:37:41.626370+0000","last_became_peered":"2026-02-20T21:37:41.626370+0000","last_unstale":"2026-02-20T21:37:43.628725+0000","last_undegraded":"2026-02-20T21:37:43.628725+0000","last_fullsized":"2026-02-20T21:37:43.628725+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:39:08.125334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050412799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628868+0000","last_change":"2026-02-20T21:37:43.629072+0000","last_active":"2026-02-20T21:37:43.628868+0000","last_peered":"2026-02-20T21:37:43.628868+0000","last_clean":"2026-02-20T21:37:43.628868+0000","last_became_active":"2026-02-20T21:37:41.626503+0000","last_became_peered":"2026-02-20T21:37:41.626503+0000","last_unstale":"2026-02-20T21:37:43.628868+0000","last_undegraded":"2026-02-20T21:37:43.628868+0000","last_fullsized":"2026-02-20T21:37:43.628868+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:55:04.979567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000697728,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628674+0000","last_change":"2026-02-20T21:37:43.628823+0000","last_active":"2026-02-20T21:37:43.628674+0000","last_peered":"2026-02-20T21:37:43.628674+0000","last_clean":"2026-02-20T21:37:43.628674+0000","last_became_active":"2026-02-20T21:37:41.626044+0000","last_became_peered":"2026-02-20T21:37:41.626044+0000","last_unstale":"2026-02-20T21:37:43.628674+0000","last_undegraded":"2026-02-20T21:37:43.628674+0000","last_fullsized":"2026-02-20T21:37:43.628674+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:43:21.886202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052464600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.627134+0000","last_change":"2026-02-20T21:37:43.627134+0000","last_active":"2026-02-20T21:37:43.627134+0000","last_peered":"2026-02-20T21:37:43.627134+0000","last_clean":"2026-02-20T21:37:43.627134+0000","last_became_active":"2026-02-20T21:37:41.626685+0000","last_became_peered":"2026-02-20T21:37:41.626685+0000","last_unstale":"2026-02-20T21:37:43.627134+0000","last_undegraded":"2026-02-20T21:37:43.627134+0000","last_fullsized":"2026-02-20T21:37:43.627134+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:19:24.951160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066038900000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.653775+0000","last_change":"2026-02-20T21:37:43.653844+0000","last_active":"2026-02-20T21:37:43.653775+0000","last_peered":"2026-02-20T21:37:43.653775+0000","last_clean":"2026-02-20T21:37:43.653775+0000","last_became_active":"2026-02-20T21:37:41.626554+0000","last_became_peered":"2026-02-20T21:37:41.626554+0000","last_unstale":"2026-02-20T21:37:43.653775+0000","last_undegraded":"2026-02-20T21:37:43.653775+0000","last_fullsized":"2026-02-20T21:37:43.653775+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:23:54.818688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021567099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.653735+0000","last_change":"2026-02-20T21:37:43.653839+0000","last_active":"2026-02-20T21:37:43.653735+0000","last_peered":"2026-02-20T21:37:43.653735+0000","last_clean":"2026-02-20T21:37:43.653735+0000","last_became_active":"2026-02-20T21:37:41.626692+0000","last_became_peered":"2026-02-20T21:37:41.626692+0000","last_unstale":"2026-02-20T21:37:43.653735+0000","last_undegraded":"2026-02-20T21:37:43.653735+0000","last_fullsized":"2026-02-20T21:37:43.653735+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:10:13.031958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021587200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628907+0000","last_change":"2026-02-20T21:37:43.629180+0000","last_active":"2026-02-20T21:37:43.628907+0000","last_peered":"2026-02-20T21:37:43.628907+0000","last_clean":"2026-02-20T21:37:43.628907+0000","last_became_active":"2026-02-20T21:37:41.626007+0000","last_became_peered":"2026-02-20T21:37:41.626007+0000","last_unstale":"2026-02-20T21:37:43.628907+0000","last_undegraded":"2026-02-20T21:37:43.628907+0000","last_fullsized":"2026-02-20T21:37:43.628907+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:01:20.786792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080498799999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628097+0000","last_change":"2026-02-20T21:37:39.618505+0000","last_active":"2026-02-20T21:37:43.628097+0000","last_peered":"2026-02-20T21:37:43.628097+0000","last_clean":"2026-02-20T21:37:43.628097+0000","last_became_active":"2026-02-20T21:37:39.618273+0000","last_became_peered":"2026-02-20T21:37:39.618273+0000","last_unstale":"2026-02-20T21:37:43.628097+0000","last_undegraded":"2026-02-20T21:37:43.628097+0000","last_fullsized":"2026-02-20T21:37:43.628097+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_clean_scrub_stamp":"2026-02-20T21:37:38.611598+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:26:23.564860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":35416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":625803,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":4,"kb_used_meta":26939,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":625803,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":4225,"internal_metadata":27586431},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:37:47.214 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:37:47.364 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:47.364 INFO:teuthology.orchestra.run.trial107.stderr:dumped all 2026-02-20T21:37:47.374 INFO:teuthology.orchestra.run.trial107.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-20T21:37:46.256763+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":81376,"kb_used_data":496,"kb_used_omap":19,"kb_used_meta":80812,"kb_avail":283034144,"statfs":{"total":289910292480,"available":289826963456,"internally_reserved":0,"allocated":507904,"data_stored":1287022,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":19825,"internal_metadata":82752143},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.630683"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.629154+0000","last_change":"2026-02-20T21:37:43.629317+0000","last_active":"2026-02-20T21:37:43.629154+0000","last_peered":"2026-02-20T21:37:43.629154+0000","last_clean":"2026-02-20T21:37:43.629154+0000","last_became_active":"2026-02-20T21:37:41.626252+0000","last_became_peered":"2026-02-20T21:37:41.626252+0000","last_unstale":"2026-02-20T21:37:43.629154+0000","last_undegraded":"2026-02-20T21:37:43.629154+0000","last_fullsized":"2026-02-20T21:37:43.629154+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:57:26.704591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000878194,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628725+0000","last_change":"2026-02-20T21:37:43.628857+0000","last_active":"2026-02-20T21:37:43.628725+0000","last_peered":"2026-02-20T21:37:43.628725+0000","last_clean":"2026-02-20T21:37:43.628725+0000","last_became_active":"2026-02-20T21:37:41.626370+0000","last_became_peered":"2026-02-20T21:37:41.626370+0000","last_unstale":"2026-02-20T21:37:43.628725+0000","last_undegraded":"2026-02-20T21:37:43.628725+0000","last_fullsized":"2026-02-20T21:37:43.628725+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:39:08.125334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050412799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628868+0000","last_change":"2026-02-20T21:37:43.629072+0000","last_active":"2026-02-20T21:37:43.628868+0000","last_peered":"2026-02-20T21:37:43.628868+0000","last_clean":"2026-02-20T21:37:43.628868+0000","last_became_active":"2026-02-20T21:37:41.626503+0000","last_became_peered":"2026-02-20T21:37:41.626503+0000","last_unstale":"2026-02-20T21:37:43.628868+0000","last_undegraded":"2026-02-20T21:37:43.628868+0000","last_fullsized":"2026-02-20T21:37:43.628868+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:55:04.979567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000697728,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628674+0000","last_change":"2026-02-20T21:37:43.628823+0000","last_active":"2026-02-20T21:37:43.628674+0000","last_peered":"2026-02-20T21:37:43.628674+0000","last_clean":"2026-02-20T21:37:43.628674+0000","last_became_active":"2026-02-20T21:37:41.626044+0000","last_became_peered":"2026-02-20T21:37:41.626044+0000","last_unstale":"2026-02-20T21:37:43.628674+0000","last_undegraded":"2026-02-20T21:37:43.628674+0000","last_fullsized":"2026-02-20T21:37:43.628674+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:43:21.886202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052464600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"14'2","reported_seq":22,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.627134+0000","last_change":"2026-02-20T21:37:43.627134+0000","last_active":"2026-02-20T21:37:43.627134+0000","last_peered":"2026-02-20T21:37:43.627134+0000","last_clean":"2026-02-20T21:37:43.627134+0000","last_became_active":"2026-02-20T21:37:41.626685+0000","last_became_peered":"2026-02-20T21:37:41.626685+0000","last_unstale":"2026-02-20T21:37:43.627134+0000","last_undegraded":"2026-02-20T21:37:43.627134+0000","last_fullsized":"2026-02-20T21:37:43.627134+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:19:24.951160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066038900000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.653775+0000","last_change":"2026-02-20T21:37:43.653844+0000","last_active":"2026-02-20T21:37:43.653775+0000","last_peered":"2026-02-20T21:37:43.653775+0000","last_clean":"2026-02-20T21:37:43.653775+0000","last_became_active":"2026-02-20T21:37:41.626554+0000","last_became_peered":"2026-02-20T21:37:41.626554+0000","last_unstale":"2026-02-20T21:37:43.653775+0000","last_undegraded":"2026-02-20T21:37:43.653775+0000","last_fullsized":"2026-02-20T21:37:43.653775+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:23:54.818688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021567099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.653735+0000","last_change":"2026-02-20T21:37:43.653839+0000","last_active":"2026-02-20T21:37:43.653735+0000","last_peered":"2026-02-20T21:37:43.653735+0000","last_clean":"2026-02-20T21:37:43.653735+0000","last_became_active":"2026-02-20T21:37:41.626692+0000","last_became_peered":"2026-02-20T21:37:41.626692+0000","last_unstale":"2026-02-20T21:37:43.653735+0000","last_undegraded":"2026-02-20T21:37:43.653735+0000","last_fullsized":"2026-02-20T21:37:43.653735+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:10:13.031958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021587200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"12'1","reported_seq":21,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628907+0000","last_change":"2026-02-20T21:37:43.629180+0000","last_active":"2026-02-20T21:37:43.628907+0000","last_peered":"2026-02-20T21:37:43.628907+0000","last_clean":"2026-02-20T21:37:43.628907+0000","last_became_active":"2026-02-20T21:37:41.626007+0000","last_became_peered":"2026-02-20T21:37:41.626007+0000","last_unstale":"2026-02-20T21:37:43.628907+0000","last_undegraded":"2026-02-20T21:37:43.628907+0000","last_fullsized":"2026-02-20T21:37:43.628907+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:01:20.786792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080498799999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":14,"state":"active+clean","last_fresh":"2026-02-20T21:37:43.628097+0000","last_change":"2026-02-20T21:37:39.618505+0000","last_active":"2026-02-20T21:37:43.628097+0000","last_peered":"2026-02-20T21:37:43.628097+0000","last_clean":"2026-02-20T21:37:43.628097+0000","last_became_active":"2026-02-20T21:37:39.618273+0000","last_became_peered":"2026-02-20T21:37:39.618273+0000","last_unstale":"2026-02-20T21:37:43.628097+0000","last_undegraded":"2026-02-20T21:37:43.628097+0000","last_fullsized":"2026-02-20T21:37:43.628097+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_clean_scrub_stamp":"2026-02-20T21:37:38.611598+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:26:23.564860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":35416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":625803,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27148,"kb_used_data":188,"kb_used_omap":4,"kb_used_meta":26939,"kb_avail":94344692,"statfs":{"total":96636764160,"available":96608964608,"internally_reserved":0,"allocated":192512,"data_stored":625803,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":4225,"internal_metadata":27586431},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:37:47.375 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-20T21:37:47.376 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-20T21:37:47.376 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-20T21:37:47.376 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-20T21:37:47.548 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T21:37:47.549 INFO:teuthology.orchestra.run.trial107.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-20T21:37:47.559 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-20T21:37:47.560 INFO:teuthology.run_tasks:Running task exec... 2026-02-20T21:37:47.577 INFO:teuthology.task.exec:Executing custom commands... 2026-02-20T21:37:47.578 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial107.front.sepia.ceph.com 2026-02-20T21:37:47.578 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:47.769 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror] 2026-02-20T21:37:47.770 INFO:teuthology.orchestra.run.trial107.stdout: key = AQD905hpMNaaNRAAw3rq8pjpJ39mjt21pn2xnQ== 2026-02-20T21:37:47.770 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-20T21:37:47.770 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:47.770 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror 2026-02-20T21:37:47.781 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:47.967 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.0] 2026-02-20T21:37:47.967 INFO:teuthology.orchestra.run.trial107.stdout: key = AQD905hpL23qORAA2psowlCFtu8YiflrIiLn1Q== 2026-02-20T21:37:47.967 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:37:47.968 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:47.968 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror.0 2026-02-20T21:37:47.978 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:48.164 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.1] 2026-02-20T21:37:48.164 INFO:teuthology.orchestra.run.trial107.stdout: key = AQD+05hpfcORAhAAoxh71EL4vMj2w6djXiLd9g== 2026-02-20T21:37:48.164 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:37:48.164 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:48.164 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror.1 2026-02-20T21:37:48.176 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:48.367 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.2] 2026-02-20T21:37:48.367 INFO:teuthology.orchestra.run.trial107.stdout: key = AQD+05hpsljbBhAAgEy41iZa4fAxkijHfKxJ4A== 2026-02-20T21:37:48.367 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:37:48.367 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:48.367 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror.2 2026-02-20T21:37:48.378 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:48.566 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.3] 2026-02-20T21:37:48.567 INFO:teuthology.orchestra.run.trial107.stdout: key = AQD+05hp4aEjCxAAE3KBsUxhRKYmTAHoTZF0ZA== 2026-02-20T21:37:48.567 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:37:48.567 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:48.567 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror.3 2026-02-20T21:37:48.577 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:48.767 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror] 2026-02-20T21:37:48.767 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpUEs5ABAA48cQEWIakhFsZFRz1Prubg== 2026-02-20T21:37:48.768 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-20T21:37:48.768 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:48.768 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror 2026-02-20T21:37:48.779 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:48.963 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.0] 2026-02-20T21:37:48.963 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpXgvABBAANWJSRLNVXHhmcRYZ6ATBPw== 2026-02-20T21:37:48.963 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:37:48.963 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:48.964 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror.0 2026-02-20T21:37:48.974 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:49.164 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.1] 2026-02-20T21:37:49.165 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpBj47CRAAbcMmIFAoKgmDR/7tXTM9JQ== 2026-02-20T21:37:49.165 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:37:49.165 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:49.165 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror.1 2026-02-20T21:37:49.176 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:49.363 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.2] 2026-02-20T21:37:49.363 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpX+vADRAAjjpWfVvdIqznng+Gx8rWbg== 2026-02-20T21:37:49.363 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:37:49.363 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:49.363 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror.2 2026-02-20T21:37:49.376 DEBUG:teuthology.orchestra.run.trial107:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:37:49.563 INFO:teuthology.orchestra.run.trial107.stdout:[client.mirror.3] 2026-02-20T21:37:49.563 INFO:teuthology.orchestra.run.trial107.stdout: key = AQAV1JhpBAT2ERAA/QNn2QUxZhUummtv4ob0/Q== 2026-02-20T21:37:49.563 INFO:teuthology.orchestra.run.trial107.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:37:49.563 INFO:teuthology.orchestra.run.trial107.stdout: caps osd = "profile rbd" 2026-02-20T21:37:49.563 INFO:teuthology.orchestra.run.trial107.stderr:updated caps for client.mirror.3 2026-02-20T21:37:49.574 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:37:49.593 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T21:37:49.593 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T21:37:49.595 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T21:37:49.595 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:37:49.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T21:37:49.618 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T21:37:49.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T21:37:49.620 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:37:49.635 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-20T21:37:49.636 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-20T21:37:49.638 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-20T21:37:49.638 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:37:49.652 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-20T21:37:49.652 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-20T21:37:49.655 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-20T21:37:49.655 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:37:49.670 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:37:49.670 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:37:49.672 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:37:49.672 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:37:49.751 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T21:37:49.752 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T21:37:49.753 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T21:37:49.753 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:37:49.764 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T21:37:49.764 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T21:37:49.765 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T21:37:49.765 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:37:49.782 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-20T21:37:49.782 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-20T21:37:49.783 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-20T21:37:49.784 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-20T21:37:49.800 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771623469 2026-02-20T21:37:49.800 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-20T21:37:49.800 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-20T21:37:49.819 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771623469 2026-02-20T21:37:49.819 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-20T21:37:49.819 INFO:teuthology.run_tasks:Running task workunit... 2026-02-20T21:37:49.837 INFO:tasks.workunit:Pulling workunits from ref f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:37:49.838 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-20T21:37:49.838 DEBUG:teuthology.orchestra.run.trial107:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:37:49.838 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2026-02-20T21:37:49.838 INFO:tasks.rbd_mirror_thrash:waiting for 116.05531656206944 secs before thrashing 2026-02-20T21:37:49.838 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2026-02-20T21:37:49.838 INFO:tasks.rbd_mirror_thrash:waiting for 91.69494047944335 secs before thrashing 2026-02-20T21:37:49.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:37:49.842 INFO:teuthology.orchestra.run.trial107.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-20T21:37:49.842 DEBUG:teuthology.orchestra.run.trial107:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:37:49.890 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:37:49.891 DEBUG:teuthology.orchestra.run.trial107:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-20T21:37:49.934 INFO:tasks.workunit:timeout=6h 2026-02-20T21:37:49.934 INFO:tasks.workunit:cleanup=True 2026-02-20T21:37:49.935 DEBUG:teuthology.orchestra.run.trial107:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:37:49.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-20T21:38:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Note: switching to 'f7870454149f22574086dd2ff55d050585136531'. 2026-02-20T21:38:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:38:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-20T21:38:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-20T21:38:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:state without impacting any branches by switching back to a branch. 2026-02-20T21:38:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:38:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: git switch -c 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Or undo this operation with: 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: git switch - 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:38:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:HEAD is now at f7870454149 ceph-dev-pipeline: configure 2026-02-20T21:38:12.112 DEBUG:teuthology.orchestra.run.trial107:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-20T21:38:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-20T21:38:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-20T21:38:12.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-20T21:38:12.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-20T21:38:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-20T21:38:12.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-20T21:38:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-20T21:38:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-20T21:38:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-20T21:38:12.342 DEBUG:teuthology.orchestra.run.trial107:> set -ex 2026-02-20T21:38:12.342 DEBUG:teuthology.orchestra.run.trial107:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-20T21:38:12.388 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2026-02-20T21:38:12.390 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2026-02-20T21:38:12.390 DEBUG:teuthology.orchestra.run.trial107:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f7870454149f22574086dd2ff55d050585136531 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=image RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-20T21:38:12.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ IMAGE_COUNT=50 2026-02-20T21:38:12.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ export LOCKDEP=0 2026-02-20T21:38:12.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ LOCKDEP=0 2026-02-20T21:38:12.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-20T21:38:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-20T21:38:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CLUSTER1=cluster1 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CLUSTER2=cluster2 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ POOL=mirror 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ PARENT_POOL=mirror_parent 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ NS1=ns1 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ NS2=ns2 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ TEMPDIR= 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ID=mirror 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,object-map,fast-diff 2026-02-20T21:38:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-20T21:38:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ RBD_MIRROR_MODE=snapshot 2026-02-20T21:38:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ MIRROR_POOL_MODE=image 2026-02-20T21:38:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' snapshot = snapshot ']' 2026-02-20T21:38:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ MIRROR_POOL_MODE=image 2026-02-20T21:38:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-20T21:38:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS='--id mirror' 2026-02-20T21:38:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-20T21:38:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-20T21:38:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-20T21:38:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_BIN=. 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_SRC=. 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 -gt 0 ']' 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ setup 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local c 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ setup_tempdir 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mktemp -d 2026-02-20T21:38:12.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ TEMPDIR=/tmp/tmp.2SC0OFePO0 2026-02-20T21:38:12.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -z 1 ']' 2026-02-20T21:38:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ setup_pools cluster1 cluster2 2026-02-20T21:38:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:38:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_cluster=cluster2 2026-02-20T21:38:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mon_map_file 2026-02-20T21:38:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mon_addr 2026-02-20T21:38:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local admin_key_file 2026-02-20T21:38:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local uuid 2026-02-20T21:38:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:12.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-20T21:38:12.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:pool 'mirror' already exists 2026-02-20T21:38:12.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:12.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-20T21:38:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:pool 'mirror_parent' already exists 2026-02-20T21:38:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:13.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-20T21:38:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-20T21:38:19.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:19.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:19.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror image 2026-02-20T21:38:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-20T21:38:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-20T21:38:19.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-20T21:38:19.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-20T21:38:19.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-02-20T21:38:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-20T21:38:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 image 2026-02-20T21:38:19.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -z ']' 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -z ']' 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ peer_add cluster1 mirror cluster2 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local client_cluster=cluster2 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_cluster=cluster2 2026-02-20T21:38:19.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local uuid_var_name 2026-02-20T21:38:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local error_code 2026-02-20T21:38:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local peer_uuid 2026-02-20T21:38:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:38:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:38:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2026-02-20T21:38:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ peer_uuid=f12817ed-2387-4f98-bc0a-136e86c08dfb 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 -eq 0 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ peer_add cluster1 mirror_parent cluster2 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror_parent 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local client_cluster=cluster2 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_cluster=cluster2 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local uuid_var_name 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local error_code 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local peer_uuid 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:38:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2026-02-20T21:38:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ peer_uuid=97f18ee1-d24b-4139-8a00-4e1135437535 2026-02-20T21:38:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 -eq 0 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ setup_pools cluster2 cluster1 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_cluster=cluster1 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mon_map_file 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mon_addr 2026-02-20T21:38:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local admin_key_file 2026-02-20T21:38:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local uuid 2026-02-20T21:38:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-20T21:38:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:pool 'mirror' already exists 2026-02-20T21:38:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-20T21:38:21.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:pool 'mirror_parent' already exists 2026-02-20T21:38:21.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:21.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-20T21:38:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:24.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-20T21:38:27.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T21:38:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror image 2026-02-20T21:38:27.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-20T21:38:27.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-20T21:38:27.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-20T21:38:27.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-20T21:38:27.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-02-20T21:38:27.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-20T21:38:27.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 image 2026-02-20T21:38:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -z ']' 2026-02-20T21:38:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -z ']' 2026-02-20T21:38:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ peer_add cluster2 mirror cluster1 2026-02-20T21:38:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:38:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local client_cluster=cluster1 2026-02-20T21:38:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_cluster=cluster1 2026-02-20T21:38:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local uuid_var_name 2026-02-20T21:38:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local error_code 2026-02-20T21:38:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local peer_uuid 2026-02-20T21:38:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:38:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:38:27.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2026-02-20T21:38:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ peer_uuid=44998845-2ed7-4998-9a42-f3f05659b0a9 2026-02-20T21:38:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T21:38:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 -eq 0 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ peer_add cluster2 mirror_parent cluster1 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror_parent 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local client_cluster=cluster1 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_cluster=cluster1 2026-02-20T21:38:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local uuid_var_name 2026-02-20T21:38:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local error_code 2026-02-20T21:38:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local peer_uuid 2026-02-20T21:38:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:38:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:38:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2026-02-20T21:38:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ peer_uuid=414cf344-34a3-46a8-9ffc-949e62e59178 2026-02-20T21:38:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T21:38:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:38:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:38:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 -eq 0 2026-02-20T21:38:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:38:27.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:38:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirrors cluster1 2026-02-20T21:38:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:38:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 0 3 2026-02-20T21:38:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirror cluster1:0 2026-02-20T21:38:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1:0 2026-02-20T21:38:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance 2026-02-20T21:38:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local val=cluster1:0 2026-02-20T21:38:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance_var_name=instance 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cluster=cluster1 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=0 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval cluster=cluster1 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval instance=0 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirror cluster1:1 2026-02-20T21:38:27.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1:1 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local val=cluster1:1 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance_var_name=instance 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cluster=cluster1 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=1 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval cluster=cluster1 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval instance=1 2026-02-20T21:38:27.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirror cluster1:2 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1:2 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local val=cluster1:2 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance_var_name=instance 2026-02-20T21:38:27.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cluster=cluster1 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=2 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval cluster=cluster1 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval instance=2 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirror cluster1:3 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1:3 2026-02-20T21:38:27.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local val=cluster1:3 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance_var_name=instance 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cluster=cluster1 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=3 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval cluster=cluster1 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval instance=3 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T21:38:27.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirrors cluster2 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 0 3 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirror cluster2:0 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2:0 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local val=cluster2:0 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance_var_name=instance 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cluster=cluster2 2026-02-20T21:38:27.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=0 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval cluster=cluster2 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster2 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval instance=0 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirror cluster2:1 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2:1 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:38:27.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local val=cluster2:1 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance_var_name=instance 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cluster=cluster2 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=1 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval cluster=cluster2 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster2 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval instance=1 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T21:38:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirror cluster2:2 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2:2 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local val=cluster2:2 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance_var_name=instance 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cluster=cluster2 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=2 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval cluster=cluster2 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster2 2026-02-20T21:38:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval instance=2 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirror cluster2:3 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2:3 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local val=cluster2:3 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance_var_name=instance 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cluster=cluster2 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=3 2026-02-20T21:38:27.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval cluster=cluster2 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster2 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval instance=3 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ tee -a /tmp/tmp.2SC0OFePO0/rbd-mirror.test.log 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ date '+%F %T' 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo 2026-02-20 21:38:27 TEST: add image and test replay after client crashes 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:2026-02-20 21:38:27 TEST: add image and test replay after client crashes 2026-02-20T21:38:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=test 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror test snapshot 512M 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror test 512M 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:38:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:27.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:38:27.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:38:27.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T21:38:27.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T21:38:27.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:38:27.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/test 2026-02-20T21:38:27.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:38:27.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:27.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:38:27.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:38:27.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror test snapshot 2026-02-20T21:38:27.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:38:27.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:27.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:38:27.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:38:27.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/test snapshot 2026-02-20T21:38:28.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:38:28.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/test 2026-02-20T21:38:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:38:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b6c8f3019f 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b6c8f3019f 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:27 2026 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:38:27 2026 2026-02-20T21:38:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:27 2026 2026-02-20T21:38:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:38:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:38:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:38:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:38:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:38:29.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:38:29.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:38:29.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:38:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:38:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:38:29.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:29.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:38:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:29.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:29.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:29.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:29.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:38:29.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:29.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:29.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:29.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:29.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:29.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:29.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:29.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:29.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:38:29.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:38:29.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:38:29.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:29.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:29.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:38:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:38:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:38:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:29.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:29.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:29.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:38:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:29.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:29.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:29.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:29.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:38:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:38:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:38:29.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:29.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:29.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 1 2026-02-20T21:38:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:38:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:38:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:38:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:30.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:30.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:30.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:30.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:38:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:38:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:38:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:38:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:38:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:38:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:38:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:38:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:38:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:38:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:38:30.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:30.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:30.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:38:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:38:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:38:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:38:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:38:30.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:38:30.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:38:30.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:30.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:30.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:30.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:30.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 2 2026-02-20T21:38:32.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:32.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:38:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:32.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:38:32.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:38:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:38:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:38:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:38:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:38:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:38:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:33.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:33.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:33.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:38:33.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:33.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:33.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:33.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:33.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:33.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:38:33.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:38:33.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:38:33.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:33.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:33.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:33.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:38:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:33.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:33.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:38:33.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:38:33.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:33.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:33.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:38:33.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:38:33.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:38:33.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:38:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:38:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 4 2026-02-20T21:38:37.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:37.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:38:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:38:37.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:37.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:37.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:37.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:37.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:37.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:37.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:37.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:37.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:38:37.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:38:37.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:37.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:38:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:38:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:38:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:38:37.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:37.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:37.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:37.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:38:37.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:38:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:38:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:38:37.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:37.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:37.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:37.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:38:37.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:38:37.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:38:37.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:37.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:37.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:37.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:38:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:37.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:37.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:38:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:38:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:38:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:38:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:38:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:38:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:37.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:37.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:37.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 8 2026-02-20T21:38:45.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:45.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:38:45.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:45.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:38:45.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:38:45.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:38:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:38:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:38:45.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:38:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:45.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:38:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:38:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:38:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:38:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:45.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:45.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:45.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:38:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:38:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:45.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:38:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:38:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:38:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:38:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:38:45.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:38:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:38:45.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:45.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:45.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:38:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:38:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:38:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:38:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:38:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:38:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:38:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:38:45.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:38:45.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:45.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:38:45.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:45.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 8 2026-02-20T21:38:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:38:53.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:53.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:38:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:38:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:38:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:38:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:38:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:53.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:53.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:53.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:53.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:38:53.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:38:53.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:38:53.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:38:53.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:38:53.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:38:53.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:38:53.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:38:53.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:38:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name= 2026-02-20T21:38:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 10 2026-02-20T21:38:53.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:38:53.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:38:53.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:38:53.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:38:53.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:38:53.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:38:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=7 2026-02-20T21:38:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:38:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:38:53.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 10094592 bytes 2026-02-20T21:38:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 57102336 bytes 2026-02-20T21:38:53.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 106106880 bytes 2026-02-20T21:38:54.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 161093632 bytes 2026-02-20T21:38:54.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 182780928 bytes 2026-02-20T21:38:54.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 237388800 bytes 2026-02-20T21:38:54.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 311892992 bytes 2026-02-20T21:38:54.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 346371072 bytes 2026-02-20T21:38:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 371461120 bytes 2026-02-20T21:38:54.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 411202560 bytes 2026-02-20T21:38:54.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 479006720 bytes 2026-02-20T21:38:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 488623104 bytes 2026-02-20T21:38:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 579385344 bytes 2026-02-20T21:38:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 643924992 bytes 2026-02-20T21:38:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 667182080 bytes 2026-02-20T21:38:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 710269952 bytes 2026-02-20T21:38:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 737121280 bytes 2026-02-20T21:38:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 774803456 bytes 2026-02-20T21:38:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 818210816 bytes 2026-02-20T21:38:54.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 863614976 bytes 2026-02-20T21:38:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 919102464 bytes 2026-02-20T21:38:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 943204352 bytes 2026-02-20T21:38:54.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1036118016 bytes 2026-02-20T21:38:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1076967424 bytes 2026-02-20T21:38:54.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1078019072 bytes 2026-02-20T21:38:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1085219840 bytes 2026-02-20T21:38:54.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1104030720 bytes 2026-02-20T21:38:54.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1126595584 bytes 2026-02-20T21:38:54.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1192253440 bytes 2026-02-20T21:38:54.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1192849408 bytes 2026-02-20T21:38:54.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1241624576 bytes 2026-02-20T21:38:54.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1252684800 bytes 2026-02-20T21:38:54.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1269686272 bytes 2026-02-20T21:38:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1311383552 bytes 2026-02-20T21:38:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1360415744 bytes 2026-02-20T21:38:54.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1386564608 bytes 2026-02-20T21:38:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1439548416 bytes 2026-02-20T21:38:55.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1512051712 bytes 2026-02-20T21:38:55.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1530636288 bytes 2026-02-20T21:38:55.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1629667328 bytes 2026-02-20T21:38:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1684770816 bytes 2026-02-20T21:38:55.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1734046720 bytes 2026-02-20T21:38:55.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1800401920 bytes 2026-02-20T21:38:55.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1833048064 bytes 2026-02-20T21:38:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1852903424 bytes 2026-02-20T21:38:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1885743104 bytes 2026-02-20T21:38:55.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1906030592 bytes 2026-02-20T21:38:55.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1969253376 bytes 2026-02-20T21:38:55.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2008582144 bytes 2026-02-20T21:38:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2059420672 bytes 2026-02-20T21:38:55.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2133228544 bytes 2026-02-20T21:38:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2254389248 bytes 2026-02-20T21:38:55.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2281647104 bytes 2026-02-20T21:38:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2297293824 bytes 2026-02-20T21:38:55.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2318873600 bytes 2026-02-20T21:38:55.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2388196352 bytes 2026-02-20T21:38:55.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2397861888 bytes 2026-02-20T21:38:55.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2457389056 bytes 2026-02-20T21:38:55.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2476641280 bytes 2026-02-20T21:38:55.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2546475008 bytes 2026-02-20T21:38:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2567376896 bytes 2026-02-20T21:38:55.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2602731520 bytes 2026-02-20T21:38:55.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2631471104 bytes 2026-02-20T21:38:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2720453632 bytes 2026-02-20T21:38:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2745736192 bytes 2026-02-20T21:38:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2773401600 bytes 2026-02-20T21:38:55.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2803706880 bytes 2026-02-20T21:38:55.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2889455616 bytes 2026-02-20T21:38:55.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2890091520 bytes 2026-02-20T21:38:55.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2937921536 bytes 2026-02-20T21:38:55.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2991765504 bytes 2026-02-20T21:38:56.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3043146752 bytes 2026-02-20T21:38:56.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3130947584 bytes 2026-02-20T21:38:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3181165568 bytes 2026-02-20T21:38:56.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3215764480 bytes 2026-02-20T21:38:56.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3225430016 bytes 2026-02-20T21:38:56.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3273051136 bytes 2026-02-20T21:38:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3360722944 bytes 2026-02-20T21:38:56.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3434187776 bytes 2026-02-20T21:38:56.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3437528064 bytes 2026-02-20T21:38:56.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3521755136 bytes 2026-02-20T21:38:56.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3613250560 bytes 2026-02-20T21:38:56.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3618098176 bytes 2026-02-20T21:38:56.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3670634496 bytes 2026-02-20T21:38:56.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3715194880 bytes 2026-02-20T21:38:56.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3788642304 bytes 2026-02-20T21:38:56.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3814935552 bytes 2026-02-20T21:38:56.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3816043520 bytes 2026-02-20T21:38:56.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3853158400 bytes 2026-02-20T21:38:56.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3892303872 bytes 2026-02-20T21:38:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3906921472 bytes 2026-02-20T21:38:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3943833600 bytes 2026-02-20T21:38:56.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4015398912 bytes 2026-02-20T21:38:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4028005376 bytes 2026-02-20T21:38:56.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4067454976 bytes 2026-02-20T21:38:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4071574528 bytes 2026-02-20T21:38:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4096966656 bytes 2026-02-20T21:38:56.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4138294272 bytes 2026-02-20T21:38:56.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4229514240 bytes 2026-02-20T21:38:56.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4308089856 bytes 2026-02-20T21:38:56.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4390029312 bytes 2026-02-20T21:38:56.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4424870912 bytes 2026-02-20T21:38:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4468995072 bytes 2026-02-20T21:38:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4514579456 bytes 2026-02-20T21:38:57.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4576160768 bytes 2026-02-20T21:38:57.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4601298944 bytes 2026-02-20T21:38:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4696186880 bytes 2026-02-20T21:38:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4765302784 bytes 2026-02-20T21:38:57.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4840523776 bytes 2026-02-20T21:38:57.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4872451072 bytes 2026-02-20T21:38:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4900748288 bytes 2026-02-20T21:38:57.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4944010240 bytes 2026-02-20T21:38:57.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4987056128 bytes 2026-02-20T21:38:57.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5044518912 bytes 2026-02-20T21:38:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5084684288 bytes 2026-02-20T21:38:57.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5115273216 bytes 2026-02-20T21:38:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5159857152 bytes 2026-02-20T21:38:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5179330560 bytes 2026-02-20T21:38:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5200893952 bytes 2026-02-20T21:38:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5223208960 bytes 2026-02-20T21:38:57.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5285451776 bytes 2026-02-20T21:38:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5313341440 bytes 2026-02-20T21:38:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5379066880 bytes 2026-02-20T21:38:57.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5396027392 bytes 2026-02-20T21:38:57.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5444251648 bytes 2026-02-20T21:38:57.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5493709824 bytes 2026-02-20T21:38:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5506942976 bytes 2026-02-20T21:38:57.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5578068992 bytes 2026-02-20T21:38:57.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5595563008 bytes 2026-02-20T21:38:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5669221376 bytes 2026-02-20T21:38:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5707557888 bytes 2026-02-20T21:38:57.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5739523072 bytes 2026-02-20T21:38:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5774238720 bytes 2026-02-20T21:38:57.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5833786368 bytes 2026-02-20T21:38:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5845349376 bytes 2026-02-20T21:38:58.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5982430208 bytes 2026-02-20T21:38:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 6011671552 bytes 2026-02-20T21:38:58.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6033051648 bytes 2026-02-20T21:38:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6089040896 bytes 2026-02-20T21:38:58.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6131468288 bytes 2026-02-20T21:38:58.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6183162880 bytes 2026-02-20T21:38:58.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6302982144 bytes 2026-02-20T21:38:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6340333568 bytes 2026-02-20T21:38:58.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6404104192 bytes 2026-02-20T21:38:58.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6448389120 bytes 2026-02-20T21:38:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6485561344 bytes 2026-02-20T21:38:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6508086272 bytes 2026-02-20T21:38:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6531677184 bytes 2026-02-20T21:38:58.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6586428416 bytes 2026-02-20T21:38:58.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6621687808 bytes 2026-02-20T21:38:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6627869696 bytes 2026-02-20T21:38:58.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6751920128 bytes 2026-02-20T21:38:58.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6752538624 bytes 2026-02-20T21:38:58.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6753174528 bytes 2026-02-20T21:38:58.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6768788480 bytes 2026-02-20T21:38:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6769401856 bytes 2026-02-20T21:38:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6808686592 bytes 2026-02-20T21:38:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6882113536 bytes 2026-02-20T21:38:58.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6938647552 bytes 2026-02-20T21:38:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6976537600 bytes 2026-02-20T21:38:58.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7010360320 bytes 2026-02-20T21:38:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7094948864 bytes 2026-02-20T21:38:58.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7144257536 bytes 2026-02-20T21:38:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7163131904 bytes 2026-02-20T21:38:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7212495872 bytes 2026-02-20T21:38:58.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7266482176 bytes 2026-02-20T21:38:58.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7289743360 bytes 2026-02-20T21:38:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7320254464 bytes 2026-02-20T21:38:59.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7351103488 bytes 2026-02-20T21:38:59.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7383356416 bytes 2026-02-20T21:38:59.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7404901376 bytes 2026-02-20T21:38:59.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7440478208 bytes 2026-02-20T21:38:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7506777088 bytes 2026-02-20T21:38:59.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7527124992 bytes 2026-02-20T21:38:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7543580672 bytes 2026-02-20T21:38:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7558170624 bytes 2026-02-20T21:38:59.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7626799104 bytes 2026-02-20T21:38:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7652625408 bytes 2026-02-20T21:38:59.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7700857856 bytes 2026-02-20T21:38:59.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7728373760 bytes 2026-02-20T21:38:59.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7749502976 bytes 2026-02-20T21:38:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7835252736 bytes 2026-02-20T21:38:59.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7854685184 bytes 2026-02-20T21:38:59.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7901278208 bytes 2026-02-20T21:38:59.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7937296384 bytes 2026-02-20T21:38:59.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7987368960 bytes 2026-02-20T21:38:59.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8021389312 bytes 2026-02-20T21:38:59.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8119836672 bytes 2026-02-20T21:38:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8162982912 bytes 2026-02-20T21:38:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8205732864 bytes 2026-02-20T21:38:59.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8249120768 bytes 2026-02-20T21:38:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8289528832 bytes 2026-02-20T21:38:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8352708608 bytes 2026-02-20T21:38:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8429484032 bytes 2026-02-20T21:38:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8504363008 bytes 2026-02-20T21:38:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8554546176 bytes 2026-02-20T21:38:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8600082432 bytes 2026-02-20T21:38:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8623226880 bytes 2026-02-20T21:39:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8639168512 bytes 2026-02-20T21:39:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8718215168 bytes 2026-02-20T21:39:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8724775936 bytes 2026-02-20T21:39:00.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8783916032 bytes 2026-02-20T21:39:00.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8867813376 bytes 2026-02-20T21:39:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8868428800 bytes 2026-02-20T21:39:00.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8869050368 bytes 2026-02-20T21:39:00.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8896101376 bytes 2026-02-20T21:39:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8899016704 bytes 2026-02-20T21:39:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8903790592 bytes 2026-02-20T21:39:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8947984384 bytes 2026-02-20T21:39:00.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9021198336 bytes 2026-02-20T21:39:00.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9062172672 bytes 2026-02-20T21:39:00.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9118151680 bytes 2026-02-20T21:39:00.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9159536640 bytes 2026-02-20T21:39:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9229477888 bytes 2026-02-20T21:39:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9276692480 bytes 2026-02-20T21:39:00.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9328893952 bytes 2026-02-20T21:39:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9384046592 bytes 2026-02-20T21:39:00.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9525468160 bytes 2026-02-20T21:39:00.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9574714368 bytes 2026-02-20T21:39:00.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9593626624 bytes 2026-02-20T21:39:00.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9652488192 bytes 2026-02-20T21:39:00.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9739622400 bytes 2026-02-20T21:39:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:39:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:39:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:39:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:39:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:39:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:39:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:39:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:39:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:39:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:39:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:01.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:39:01.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:39:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:39:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:39:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:39:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771623508,"replay_state":"idle"} 2026-02-20T21:39:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.0 on trial107 2026-02-20T21:39:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:38:51 2026-02-20T21:39:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:39:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:39:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: down+unknown 2026-02-20T21:39:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: status not found 2026-02-20T21:39:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: ' 2026-02-20T21:39:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:39:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:39:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:39:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771623508,"replay_state":"idle"} 2026-02-20T21:39:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:39:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:39:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:39:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:39:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:39:01.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.0 on trial107 2026-02-20T21:39:01.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:39:01.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:39:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:38:51' 2026-02-20T21:39:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:38:51' 2026-02-20T21:39:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:01.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:39:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:38:51' 2026-02-20T21:39:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:39:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:39:01.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:39:01.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='WARNINGOKWARNING142114264mirror.0trial10720.2.0-677-gf7870454trueOK42154266mirror.1trial10720.2.0-677-gf7870454falseOK42194269mirror.2trial10720.2.0-677-gf7870454falseOK42234272mirror.3trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771623508,"replay_state":"idle"}42114264mirror.0trial1072026-02-20 21:38:51cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911ddown+unknownstatus not found' 2026-02-20T21:39:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:39:01.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:39:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:38:51' 2026-02-20T21:39:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:39:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:39:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:39:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:39:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:39:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap1 2026-02-20T21:39:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap1 2026-02-20T21:39:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:39:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap1 2026-02-20T21:39:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:39:21.534 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7739693921294759) > weight (0.25) 2026-02-20T21:39:21.534 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-20T21:39:21.535 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-20T21:39:21.535 INFO:tasks.rbd_mirror_thrash:waiting for 9.956570681016801 secs before reviving daemons 2026-02-20T21:39:21.536 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T21:39:21.533+0000 7fa4858b7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 19460) UID: 1000 2026-02-20T21:39:31.493 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-20T21:39:31.493 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:39:31.494 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-20T21:39:31.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T21:39:31.494 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T21:39:31.496 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T21:39:31.496 INFO:tasks.rbd_mirror_thrash:waiting for 108.8032674353372 secs before thrashing 2026-02-20T21:39:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:39:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:39:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:39:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:39:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:39:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:39:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:39:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:39:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:39:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:39:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:39:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:39:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:39:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:39:33.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:39:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:39:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:39:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:39:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:39:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:39:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:39:33.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:39:33.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:39:33.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:39:33.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:39:33.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:39:33.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:39:33.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:39:33.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:39:33.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:39:33.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:39:33.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:39:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:39:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:39:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:39:34.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 5 2026-02-20T21:39:34.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:39:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:39:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:39:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:39:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:39:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:39:34.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:39:34.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:39:34.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:39:34.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:39:34.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:39:34.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:39:34.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:39:34.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:39:34.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:39:34.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:34.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:34.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:39:34.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:39:34.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:39:34.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:39:34.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=5 2026-02-20T21:39:34.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:39:34.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:39:34.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:39:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:39:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:39:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:39:34.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:39:34.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:39:34.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:39:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:39:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:39:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:39:34.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:39:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 5 = '' 2026-02-20T21:39:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:39:34.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:39:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:39:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:39:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:39:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:39:35.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:39:35.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:39:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:39:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:39:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:39:35.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:39:35.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 5 = '' 2026-02-20T21:39:35.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:39:35.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:39:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:39:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:39:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:39:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:39:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:39:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=5 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 5 = 5 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap1 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:39:36.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:39:37.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2026-02-20T21:39:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:39:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:39:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:39:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:39:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:39:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:39:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:39:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:39:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:39:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap1 2026-02-20T21:39:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap1 2026-02-20T21:39:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:39:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:39:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:39:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:39:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:39:37.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:39:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:39:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:39:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:39:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:39:38.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:38.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap1-clean 2026-02-20T21:39:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap1-clean 2026-02-20T21:39:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:39:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap1-clean 2026-02-20T21:39:38.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:39:39.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:39:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:39:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:39.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:39:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=8 2026-02-20T21:39:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:39:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:39:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 118062080 bytes 2026-02-20T21:39:39.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 134183936 bytes 2026-02-20T21:39:39.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 174759936 bytes 2026-02-20T21:39:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 226873344 bytes 2026-02-20T21:39:39.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 268050432 bytes 2026-02-20T21:39:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 323938304 bytes 2026-02-20T21:39:40.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 349953024 bytes 2026-02-20T21:39:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 391024640 bytes 2026-02-20T21:39:40.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 464704512 bytes 2026-02-20T21:39:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 492972032 bytes 2026-02-20T21:39:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 551948288 bytes 2026-02-20T21:39:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 632045568 bytes 2026-02-20T21:39:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 651234304 bytes 2026-02-20T21:39:40.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 697991168 bytes 2026-02-20T21:39:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 743660544 bytes 2026-02-20T21:39:40.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 763386880 bytes 2026-02-20T21:39:40.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 824510464 bytes 2026-02-20T21:39:40.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 868197376 bytes 2026-02-20T21:39:40.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 885936128 bytes 2026-02-20T21:39:40.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 931345408 bytes 2026-02-20T21:39:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 948546560 bytes 2026-02-20T21:39:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 984689664 bytes 2026-02-20T21:39:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1041673216 bytes 2026-02-20T21:39:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1093660672 bytes 2026-02-20T21:39:40.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1127622656 bytes 2026-02-20T21:39:40.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1182270464 bytes 2026-02-20T21:39:40.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1258210304 bytes 2026-02-20T21:39:40.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1314351104 bytes 2026-02-20T21:39:40.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1351519232 bytes 2026-02-20T21:39:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1378590720 bytes 2026-02-20T21:39:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1437095936 bytes 2026-02-20T21:39:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1510602752 bytes 2026-02-20T21:39:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1598078976 bytes 2026-02-20T21:39:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1669511168 bytes 2026-02-20T21:39:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1670122496 bytes 2026-02-20T21:39:41.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1739231232 bytes 2026-02-20T21:39:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1791232000 bytes 2026-02-20T21:39:41.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1821712384 bytes 2026-02-20T21:39:41.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1879335936 bytes 2026-02-20T21:39:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1937040384 bytes 2026-02-20T21:39:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1974203392 bytes 2026-02-20T21:39:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1982484480 bytes 2026-02-20T21:39:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1987947520 bytes 2026-02-20T21:39:41.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2029954048 bytes 2026-02-20T21:39:41.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2076789760 bytes 2026-02-20T21:39:41.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2124923904 bytes 2026-02-20T21:39:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2125533184 bytes 2026-02-20T21:39:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2156891136 bytes 2026-02-20T21:39:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2202575872 bytes 2026-02-20T21:39:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2275571712 bytes 2026-02-20T21:39:41.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2341060608 bytes 2026-02-20T21:39:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2353959936 bytes 2026-02-20T21:39:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2386666496 bytes 2026-02-20T21:39:41.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2452340736 bytes 2026-02-20T21:39:41.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2558655488 bytes 2026-02-20T21:39:41.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2583742464 bytes 2026-02-20T21:39:41.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2621297664 bytes 2026-02-20T21:39:41.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2688926720 bytes 2026-02-20T21:39:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2731131904 bytes 2026-02-20T21:39:41.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2793106432 bytes 2026-02-20T21:39:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2816062464 bytes 2026-02-20T21:39:41.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2896245760 bytes 2026-02-20T21:39:41.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2927987712 bytes 2026-02-20T21:39:41.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2989105152 bytes 2026-02-20T21:39:41.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3004960768 bytes 2026-02-20T21:39:41.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3033750528 bytes 2026-02-20T21:39:42.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3091458048 bytes 2026-02-20T21:39:42.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3142529024 bytes 2026-02-20T21:39:42.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3178971136 bytes 2026-02-20T21:39:42.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3240721408 bytes 2026-02-20T21:39:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3252071424 bytes 2026-02-20T21:39:42.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3288097792 bytes 2026-02-20T21:39:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3367147520 bytes 2026-02-20T21:39:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3417371648 bytes 2026-02-20T21:39:42.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3467100160 bytes 2026-02-20T21:39:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3521582080 bytes 2026-02-20T21:39:42.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3548181504 bytes 2026-02-20T21:39:42.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3577016320 bytes 2026-02-20T21:39:42.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3592861696 bytes 2026-02-20T21:39:42.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3616836608 bytes 2026-02-20T21:39:42.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3657197568 bytes 2026-02-20T21:39:42.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3707589632 bytes 2026-02-20T21:39:42.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3756921856 bytes 2026-02-20T21:39:42.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3757535232 bytes 2026-02-20T21:39:42.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3772604416 bytes 2026-02-20T21:39:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3797871616 bytes 2026-02-20T21:39:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3809993728 bytes 2026-02-20T21:39:42.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3866361856 bytes 2026-02-20T21:39:42.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3905049600 bytes 2026-02-20T21:39:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3921977344 bytes 2026-02-20T21:39:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3991422976 bytes 2026-02-20T21:39:42.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4036647936 bytes 2026-02-20T21:39:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4051974144 bytes 2026-02-20T21:39:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4124448768 bytes 2026-02-20T21:39:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4192587776 bytes 2026-02-20T21:39:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4263091200 bytes 2026-02-20T21:39:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4315643904 bytes 2026-02-20T21:39:43.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4339719168 bytes 2026-02-20T21:39:43.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4415748096 bytes 2026-02-20T21:39:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4496025600 bytes 2026-02-20T21:39:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4526741504 bytes 2026-02-20T21:39:43.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4575122432 bytes 2026-02-20T21:39:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4618158080 bytes 2026-02-20T21:39:43.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4669017088 bytes 2026-02-20T21:39:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4722282496 bytes 2026-02-20T21:39:43.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4798557184 bytes 2026-02-20T21:39:43.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4824665088 bytes 2026-02-20T21:39:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4838004736 bytes 2026-02-20T21:39:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4892021760 bytes 2026-02-20T21:39:43.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4925558784 bytes 2026-02-20T21:39:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5011160064 bytes 2026-02-20T21:39:43.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5056349184 bytes 2026-02-20T21:39:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5097391104 bytes 2026-02-20T21:39:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5145227264 bytes 2026-02-20T21:39:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5148788736 bytes 2026-02-20T21:39:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5166348288 bytes 2026-02-20T21:39:43.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5215845376 bytes 2026-02-20T21:39:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5262384128 bytes 2026-02-20T21:39:43.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5313998848 bytes 2026-02-20T21:39:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5385684992 bytes 2026-02-20T21:39:43.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5437041664 bytes 2026-02-20T21:39:43.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5470116864 bytes 2026-02-20T21:39:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5548532736 bytes 2026-02-20T21:39:43.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5574715392 bytes 2026-02-20T21:39:44.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5628915712 bytes 2026-02-20T21:39:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5687316480 bytes 2026-02-20T21:39:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5769618432 bytes 2026-02-20T21:39:44.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5831489536 bytes 2026-02-20T21:39:44.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5851895808 bytes 2026-02-20T21:39:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5913008128 bytes 2026-02-20T21:39:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5970336768 bytes 2026-02-20T21:39:44.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5994654720 bytes 2026-02-20T21:39:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 6062246912 bytes 2026-02-20T21:39:44.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 6115312640 bytes 2026-02-20T21:39:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 6201292800 bytes 2026-02-20T21:39:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 6230185984 bytes 2026-02-20T21:39:44.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 6248527872 bytes 2026-02-20T21:39:44.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6321125376 bytes 2026-02-20T21:39:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6367891456 bytes 2026-02-20T21:39:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6437296128 bytes 2026-02-20T21:39:44.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6456512512 bytes 2026-02-20T21:39:44.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6499329024 bytes 2026-02-20T21:39:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6521214976 bytes 2026-02-20T21:39:44.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6582678528 bytes 2026-02-20T21:39:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6596323328 bytes 2026-02-20T21:39:44.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6661787648 bytes 2026-02-20T21:39:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6697518080 bytes 2026-02-20T21:39:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6698135552 bytes 2026-02-20T21:39:44.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6728986624 bytes 2026-02-20T21:39:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6769828864 bytes 2026-02-20T21:39:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6816106496 bytes 2026-02-20T21:39:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6876872704 bytes 2026-02-20T21:39:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6884061184 bytes 2026-02-20T21:39:45.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6966660096 bytes 2026-02-20T21:39:45.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6967495680 bytes 2026-02-20T21:39:45.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6997464064 bytes 2026-02-20T21:39:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 7050074112 bytes 2026-02-20T21:39:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 7082675200 bytes 2026-02-20T21:39:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 7129196544 bytes 2026-02-20T21:39:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 7151614976 bytes 2026-02-20T21:39:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7191558144 bytes 2026-02-20T21:39:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7224844288 bytes 2026-02-20T21:39:45.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7248079872 bytes 2026-02-20T21:39:45.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7281275904 bytes 2026-02-20T21:39:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7346204672 bytes 2026-02-20T21:39:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7386307584 bytes 2026-02-20T21:39:45.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7423888384 bytes 2026-02-20T21:39:45.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7481585664 bytes 2026-02-20T21:39:45.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7493770240 bytes 2026-02-20T21:39:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7527201792 bytes 2026-02-20T21:39:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7597465600 bytes 2026-02-20T21:39:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7664281600 bytes 2026-02-20T21:39:45.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7721693184 bytes 2026-02-20T21:39:45.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7737804800 bytes 2026-02-20T21:39:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7775276032 bytes 2026-02-20T21:39:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7862381568 bytes 2026-02-20T21:39:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7923371008 bytes 2026-02-20T21:39:45.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7948537856 bytes 2026-02-20T21:39:45.895 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.755471237987742) > weight (0.25) 2026-02-20T21:39:45.895 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.37540985480634725) > weight (0.25) 2026-02-20T21:39:45.895 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.692572336690165) > weight (0.25) 2026-02-20T21:39:45.895 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7279699297622568) > weight (0.25) 2026-02-20T21:39:45.895 INFO:tasks.rbd_mirror_thrash:waiting for 100.56307303258754 secs before thrashing 2026-02-20T21:39:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 8027653120 bytes 2026-02-20T21:39:45.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 8079383552 bytes 2026-02-20T21:39:45.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 8126220288 bytes 2026-02-20T21:39:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 8188830720 bytes 2026-02-20T21:39:46.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 8297251840 bytes 2026-02-20T21:39:46.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 8338780160 bytes 2026-02-20T21:39:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 8429834240 bytes 2026-02-20T21:39:46.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 8530632704 bytes 2026-02-20T21:39:46.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8583710720 bytes 2026-02-20T21:39:46.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8633417728 bytes 2026-02-20T21:39:46.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8704183296 bytes 2026-02-20T21:39:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8753345536 bytes 2026-02-20T21:39:46.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8778448896 bytes 2026-02-20T21:39:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8825613312 bytes 2026-02-20T21:39:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8876919808 bytes 2026-02-20T21:39:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8898867200 bytes 2026-02-20T21:39:46.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8928919552 bytes 2026-02-20T21:39:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8968162304 bytes 2026-02-20T21:39:46.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8999249920 bytes 2026-02-20T21:39:46.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 9046980608 bytes 2026-02-20T21:39:46.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 9106772992 bytes 2026-02-20T21:39:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 9146969088 bytes 2026-02-20T21:39:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 9196133376 bytes 2026-02-20T21:39:46.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 9265614848 bytes 2026-02-20T21:39:46.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 9322563584 bytes 2026-02-20T21:39:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 9384730624 bytes 2026-02-20T21:39:47.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 9444630528 bytes 2026-02-20T21:39:47.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 9487664128 bytes 2026-02-20T21:39:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 9535522816 bytes 2026-02-20T21:39:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 9576171520 bytes 2026-02-20T21:39:47.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 9594592256 bytes 2026-02-20T21:39:47.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9643796480 bytes 2026-02-20T21:39:47.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9721954304 bytes 2026-02-20T21:39:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9739529216 bytes 2026-02-20T21:39:47.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9798081536 bytes 2026-02-20T21:39:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9875493888 bytes 2026-02-20T21:39:47.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9906134016 bytes 2026-02-20T21:39:47.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9930352640 bytes 2026-02-20T21:39:47.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9944691712 bytes 2026-02-20T21:39:47.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9986550784 bytes 2026-02-20T21:39:47.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:39:47.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:39:47.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:39:47.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:47.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:39:47.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:39:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:39:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:39:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:39:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:39:48.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:39:48.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:48.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:48.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:48.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:39:48.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:39:48.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:39:48.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:39:48.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:39:48.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:39:48.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623508,"remote_snapshot_timestamp":1771623508,"replay_state":"idle"} 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.0 on trial107 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:39:19 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+stopped 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: local image is primary 2026-02-20T21:39:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:39:19' 2026-02-20T21:39:48.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:39:48.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:39:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:39:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623508,"remote_snapshot_timestamp":1771623508,"replay_state":"idle"} 2026-02-20T21:39:48.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:39:48.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:39:48.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:39:48.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:39:48.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:39:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.0 on trial107 2026-02-20T21:39:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:39:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:39:48.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:39:19' 2026-02-20T21:39:48.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:39:19' 2026-02-20T21:39:48.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:39:48.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:48.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:48.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:39:48.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:39:19' 2026-02-20T21:39:48.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:39:48.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:39:48.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:39:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='OKOKOK142114264mirror.0trial10720.2.0-677-gf7870454trueOK42154266mirror.1trial10720.2.0-677-gf7870454falseOK42194269mirror.2trial10720.2.0-677-gf7870454falseOK42234272mirror.3trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623508,"remote_snapshot_timestamp":1771623508,"replay_state":"idle"}42114264mirror.0trial1072026-02-20 21:39:19cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911dup+stoppedlocal image is primary2026-02-20 21:39:19' 2026-02-20T21:39:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:39:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:39:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:39:19' 2026-02-20T21:39:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:39:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:39:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:39:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:39:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:39:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:39:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap2 2026-02-20T21:39:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap2 2026-02-20T21:39:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:39:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:39:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:39:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap2 2026-02-20T21:39:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:40:18.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:40:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:40:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:40:18.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:40:18.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:40:18.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:40:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:40:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:40:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:40:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:40:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:40:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:40:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:40:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:40:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:40:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:40:19.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:40:19.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:40:19.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:40:19.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:40:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:40:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:40:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:40:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:40:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:40:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:40:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:40:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:40:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:40:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:40:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:40:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:40:19.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:40:19.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:40:19.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:40:19.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:40:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:40:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:40:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:40:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:40:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:40:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:40:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:40:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:40:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:40:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:40:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:40:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:40:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:40:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:40:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:40:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:40:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:40:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:19.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:40:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 8 2026-02-20T21:40:21.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:40:21.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:40:21.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:40:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:40:21.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:40:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:40:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:40:21.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:40:21.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:40:21.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:40:21.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:40:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:40:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:40:21.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:40:21.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:40:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=8 2026-02-20T21:40:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:40:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:40:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:40:21.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:40:21.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:21.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:21.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:21.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:40:21.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:40:21.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:40:21.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:40:21.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:40:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:40:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:40:21.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:40:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:40:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 8 = '' 2026-02-20T21:40:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:40:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:40:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:40:22.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:40:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:40:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:40:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:40:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:40:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:40:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:40:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:40:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:40:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 8 = '' 2026-02-20T21:40:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:40:22.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:40:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:40:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:40:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:40:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:40:23.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=8 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 8 = 8 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap2 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:40:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:40:23.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:40:24.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2026-02-20T21:40:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:40:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:40:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:40:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 4 2026-02-20T21:40:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:40:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:40:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:40:24.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:40:24.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:40:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:40:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snap1-clean ']' 2026-02-20T21:40:24.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-20T21:40:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap1-clean 2026-02-20T21:40:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:40:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:40:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:24.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:40:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:40:24.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:40:25.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:40:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap2 2026-02-20T21:40:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap2 2026-02-20T21:40:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:40:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:40:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:40:26.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:40:26.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:26.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:40:26.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:27.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:40:27.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:40:27.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap2-clean 2026-02-20T21:40:27.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap2-clean 2026-02-20T21:40:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:40:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap2-clean 2026-02-20T21:40:27.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:40:27.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:40:27.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:40:27.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:40:27.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:27.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:27.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:40:27.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T21:40:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:40:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:40:27.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 46922752 bytes 2026-02-20T21:40:27.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 47534080 bytes 2026-02-20T21:40:27.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 48144384 bytes 2026-02-20T21:40:27.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 74045440 bytes 2026-02-20T21:40:27.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 145956864 bytes 2026-02-20T21:40:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 193455104 bytes 2026-02-20T21:40:27.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 248517632 bytes 2026-02-20T21:40:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 303940608 bytes 2026-02-20T21:40:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 346112000 bytes 2026-02-20T21:40:28.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 355124224 bytes 2026-02-20T21:40:28.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 378988544 bytes 2026-02-20T21:40:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 431219712 bytes 2026-02-20T21:40:28.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 468746240 bytes 2026-02-20T21:40:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 550664192 bytes 2026-02-20T21:40:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 594613248 bytes 2026-02-20T21:40:28.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 689760256 bytes 2026-02-20T21:40:28.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 715220992 bytes 2026-02-20T21:40:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 784476160 bytes 2026-02-20T21:40:28.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 862876672 bytes 2026-02-20T21:40:28.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 882366464 bytes 2026-02-20T21:40:28.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 930251776 bytes 2026-02-20T21:40:28.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 958745600 bytes 2026-02-20T21:40:28.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 977971200 bytes 2026-02-20T21:40:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1022392320 bytes 2026-02-20T21:40:28.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1098718208 bytes 2026-02-20T21:40:28.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1099341824 bytes 2026-02-20T21:40:28.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1131850752 bytes 2026-02-20T21:40:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1146726400 bytes 2026-02-20T21:40:28.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1171054592 bytes 2026-02-20T21:40:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1257570304 bytes 2026-02-20T21:40:28.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1279427584 bytes 2026-02-20T21:40:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1306689536 bytes 2026-02-20T21:40:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1355601920 bytes 2026-02-20T21:40:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1384974336 bytes 2026-02-20T21:40:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1442791424 bytes 2026-02-20T21:40:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1505477632 bytes 2026-02-20T21:40:29.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1526463488 bytes 2026-02-20T21:40:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1564861440 bytes 2026-02-20T21:40:29.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1646021632 bytes 2026-02-20T21:40:29.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1660465152 bytes 2026-02-20T21:40:29.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1704098816 bytes 2026-02-20T21:40:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1782365184 bytes 2026-02-20T21:40:29.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1807060992 bytes 2026-02-20T21:40:29.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1837491200 bytes 2026-02-20T21:40:29.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1859991552 bytes 2026-02-20T21:40:29.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1917667328 bytes 2026-02-20T21:40:29.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1945121792 bytes 2026-02-20T21:40:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1987917824 bytes 2026-02-20T21:40:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2033728512 bytes 2026-02-20T21:40:29.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2093414400 bytes 2026-02-20T21:40:29.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2149014528 bytes 2026-02-20T21:40:29.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2176079872 bytes 2026-02-20T21:40:29.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2220624896 bytes 2026-02-20T21:40:29.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2266678272 bytes 2026-02-20T21:40:29.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2302673920 bytes 2026-02-20T21:40:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2313177088 bytes 2026-02-20T21:40:29.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2390019072 bytes 2026-02-20T21:40:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2427369472 bytes 2026-02-20T21:40:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2441492480 bytes 2026-02-20T21:40:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2499903488 bytes 2026-02-20T21:40:30.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2506366976 bytes 2026-02-20T21:40:30.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2525680640 bytes 2026-02-20T21:40:30.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2591514624 bytes 2026-02-20T21:40:30.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2680052736 bytes 2026-02-20T21:40:30.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2733200384 bytes 2026-02-20T21:40:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2748048384 bytes 2026-02-20T21:40:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2784977920 bytes 2026-02-20T21:40:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2797519872 bytes 2026-02-20T21:40:30.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2798130176 bytes 2026-02-20T21:40:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2843118592 bytes 2026-02-20T21:40:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2888853504 bytes 2026-02-20T21:40:30.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2891641856 bytes 2026-02-20T21:40:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2955033600 bytes 2026-02-20T21:40:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2991816704 bytes 2026-02-20T21:40:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3012474880 bytes 2026-02-20T21:40:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3045769216 bytes 2026-02-20T21:40:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3094660096 bytes 2026-02-20T21:40:30.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3123171328 bytes 2026-02-20T21:40:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3175417856 bytes 2026-02-20T21:40:30.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3208775680 bytes 2026-02-20T21:40:30.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3247232000 bytes 2026-02-20T21:40:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3288465408 bytes 2026-02-20T21:40:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3294192640 bytes 2026-02-20T21:40:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3341473792 bytes 2026-02-20T21:40:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3352596480 bytes 2026-02-20T21:40:30.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3415751680 bytes 2026-02-20T21:40:30.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3452702720 bytes 2026-02-20T21:40:30.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3516050432 bytes 2026-02-20T21:40:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3530000384 bytes 2026-02-20T21:40:31.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3565803520 bytes 2026-02-20T21:40:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3591177216 bytes 2026-02-20T21:40:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3613543424 bytes 2026-02-20T21:40:31.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3661403136 bytes 2026-02-20T21:40:31.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3740659712 bytes 2026-02-20T21:40:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3775992832 bytes 2026-02-20T21:40:31.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3817783296 bytes 2026-02-20T21:40:31.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3891438592 bytes 2026-02-20T21:40:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3916294144 bytes 2026-02-20T21:40:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3958053888 bytes 2026-02-20T21:40:31.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4005118976 bytes 2026-02-20T21:40:31.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4072299520 bytes 2026-02-20T21:40:31.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4085116928 bytes 2026-02-20T21:40:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4148035584 bytes 2026-02-20T21:40:31.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4172192768 bytes 2026-02-20T21:40:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4189944832 bytes 2026-02-20T21:40:31.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4252509184 bytes 2026-02-20T21:40:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4253132800 bytes 2026-02-20T21:40:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4259031040 bytes 2026-02-20T21:40:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4281231360 bytes 2026-02-20T21:40:31.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4281929728 bytes 2026-02-20T21:40:31.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4359927808 bytes 2026-02-20T21:40:31.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4447668224 bytes 2026-02-20T21:40:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4485603328 bytes 2026-02-20T21:40:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4508562432 bytes 2026-02-20T21:40:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4528349184 bytes 2026-02-20T21:40:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4545229824 bytes 2026-02-20T21:40:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4579907584 bytes 2026-02-20T21:40:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4611838976 bytes 2026-02-20T21:40:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4637508608 bytes 2026-02-20T21:40:32.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4670416896 bytes 2026-02-20T21:40:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4723822592 bytes 2026-02-20T21:40:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4807069696 bytes 2026-02-20T21:40:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4842079232 bytes 2026-02-20T21:40:32.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4869236736 bytes 2026-02-20T21:40:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4872014848 bytes 2026-02-20T21:40:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 4910331904 bytes 2026-02-20T21:40:32.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 4930934784 bytes 2026-02-20T21:40:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 4959516672 bytes 2026-02-20T21:40:32.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5004882944 bytes 2026-02-20T21:40:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5067308032 bytes 2026-02-20T21:40:32.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5096421376 bytes 2026-02-20T21:40:32.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5103485952 bytes 2026-02-20T21:40:32.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5104547840 bytes 2026-02-20T21:40:32.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5167215616 bytes 2026-02-20T21:40:32.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5195333632 bytes 2026-02-20T21:40:32.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5245619200 bytes 2026-02-20T21:40:32.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5246625792 bytes 2026-02-20T21:40:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5268819968 bytes 2026-02-20T21:40:32.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5293130752 bytes 2026-02-20T21:40:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5381747712 bytes 2026-02-20T21:40:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5433554944 bytes 2026-02-20T21:40:32.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5475121152 bytes 2026-02-20T21:40:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5480738816 bytes 2026-02-20T21:40:32.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5541070848 bytes 2026-02-20T21:40:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5579390976 bytes 2026-02-20T21:40:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5664618496 bytes 2026-02-20T21:40:32.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5682683904 bytes 2026-02-20T21:40:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5719189504 bytes 2026-02-20T21:40:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5725347840 bytes 2026-02-20T21:40:33.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5773471744 bytes 2026-02-20T21:40:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 5839069184 bytes 2026-02-20T21:40:33.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 5876234240 bytes 2026-02-20T21:40:33.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 5910623232 bytes 2026-02-20T21:40:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 5911227392 bytes 2026-02-20T21:40:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 5947261952 bytes 2026-02-20T21:40:33.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6033542144 bytes 2026-02-20T21:40:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6109042688 bytes 2026-02-20T21:40:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6131719168 bytes 2026-02-20T21:40:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6159936512 bytes 2026-02-20T21:40:33.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6176716800 bytes 2026-02-20T21:40:33.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6250784768 bytes 2026-02-20T21:40:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6297999360 bytes 2026-02-20T21:40:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6336864256 bytes 2026-02-20T21:40:33.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6366524416 bytes 2026-02-20T21:40:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6431576064 bytes 2026-02-20T21:40:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6452168704 bytes 2026-02-20T21:40:33.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6534310912 bytes 2026-02-20T21:40:33.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6589147136 bytes 2026-02-20T21:40:33.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6621996032 bytes 2026-02-20T21:40:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6622623744 bytes 2026-02-20T21:40:33.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6666683392 bytes 2026-02-20T21:40:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6672067584 bytes 2026-02-20T21:40:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 6697411584 bytes 2026-02-20T21:40:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 6715486208 bytes 2026-02-20T21:40:33.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 6730050560 bytes 2026-02-20T21:40:33.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 6806575104 bytes 2026-02-20T21:40:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 6861801472 bytes 2026-02-20T21:40:33.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 6891779072 bytes 2026-02-20T21:40:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 6947460096 bytes 2026-02-20T21:40:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7006845952 bytes 2026-02-20T21:40:34.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7103218688 bytes 2026-02-20T21:40:34.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7114886144 bytes 2026-02-20T21:40:34.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7161003008 bytes 2026-02-20T21:40:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7183137792 bytes 2026-02-20T21:40:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7231512576 bytes 2026-02-20T21:40:34.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7269455872 bytes 2026-02-20T21:40:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7327785984 bytes 2026-02-20T21:40:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7356809216 bytes 2026-02-20T21:40:34.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7400999936 bytes 2026-02-20T21:40:34.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7419843584 bytes 2026-02-20T21:40:34.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7506617344 bytes 2026-02-20T21:40:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7507217408 bytes 2026-02-20T21:40:34.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7557595136 bytes 2026-02-20T21:40:34.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 7569057792 bytes 2026-02-20T21:40:34.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 7611100160 bytes 2026-02-20T21:40:34.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 7627790336 bytes 2026-02-20T21:40:34.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 7665852416 bytes 2026-02-20T21:40:34.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 7763320832 bytes 2026-02-20T21:40:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 7814686720 bytes 2026-02-20T21:40:34.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 7877712896 bytes 2026-02-20T21:40:34.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 7951126528 bytes 2026-02-20T21:40:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 7951773696 bytes 2026-02-20T21:40:34.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 7972586496 bytes 2026-02-20T21:40:34.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8017523712 bytes 2026-02-20T21:40:34.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8039435264 bytes 2026-02-20T21:40:34.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8040047616 bytes 2026-02-20T21:40:35.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8124792832 bytes 2026-02-20T21:40:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8145396736 bytes 2026-02-20T21:40:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8186185728 bytes 2026-02-20T21:40:35.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8253068288 bytes 2026-02-20T21:40:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8325079040 bytes 2026-02-20T21:40:35.225 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8349685760 bytes 2026-02-20T21:40:35.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8382514176 bytes 2026-02-20T21:40:35.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8424825856 bytes 2026-02-20T21:40:35.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8426724352 bytes 2026-02-20T21:40:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 8493460480 bytes 2026-02-20T21:40:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 8537971712 bytes 2026-02-20T21:40:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 8582149120 bytes 2026-02-20T21:40:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 8663992320 bytes 2026-02-20T21:40:35.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 8721302528 bytes 2026-02-20T21:40:35.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 8770717696 bytes 2026-02-20T21:40:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 8771329024 bytes 2026-02-20T21:40:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 8793880576 bytes 2026-02-20T21:40:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 8832839680 bytes 2026-02-20T21:40:35.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 8853457920 bytes 2026-02-20T21:40:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 8936801280 bytes 2026-02-20T21:40:35.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9045936128 bytes 2026-02-20T21:40:35.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9090411520 bytes 2026-02-20T21:40:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9139409920 bytes 2026-02-20T21:40:35.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9188489216 bytes 2026-02-20T21:40:35.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9202980864 bytes 2026-02-20T21:40:35.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9258056704 bytes 2026-02-20T21:40:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9266239488 bytes 2026-02-20T21:40:36.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9285576704 bytes 2026-02-20T21:40:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9313495040 bytes 2026-02-20T21:40:36.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 9337032704 bytes 2026-02-20T21:40:36.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 9413312512 bytes 2026-02-20T21:40:36.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 9459347456 bytes 2026-02-20T21:40:36.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 9495356416 bytes 2026-02-20T21:40:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 9535263744 bytes 2026-02-20T21:40:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 9555227648 bytes 2026-02-20T21:40:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 9570007040 bytes 2026-02-20T21:40:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 9570626560 bytes 2026-02-20T21:40:36.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 9593268224 bytes 2026-02-20T21:40:36.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 9647424512 bytes 2026-02-20T21:40:36.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 9687980032 bytes 2026-02-20T21:40:36.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 9705684992 bytes 2026-02-20T21:40:36.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 9756154880 bytes 2026-02-20T21:40:36.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 9795807232 bytes 2026-02-20T21:40:36.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 9877062656 bytes 2026-02-20T21:40:36.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:40:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:40:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:40:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:40:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:40:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:40:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:40:36.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:40:36.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:40:36.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:40:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:40:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:40:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:40:37.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:40:37.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:40:37.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:40:37.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:40:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:40:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:40:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:40:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":268285440.0,"last_snapshot_bytes":536570880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623574,"remote_snapshot_timestamp":1771623574,"replay_state":"idle"} 2026-02-20T21:40:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.0 on trial107 2026-02-20T21:40:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:40:19 2026-02-20T21:40:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:40:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:40:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+stopped 2026-02-20T21:40:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: local image is primary 2026-02-20T21:40:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:40:19' 2026-02-20T21:40:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:40:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:40:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:40:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":268285440.0,"last_snapshot_bytes":536570880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623574,"remote_snapshot_timestamp":1771623574,"replay_state":"idle"} 2026-02-20T21:40:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:40:37.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:40:37.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:40:37.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:40:37.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:40:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.0 on trial107 2026-02-20T21:40:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:40:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:40:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:40:19' 2026-02-20T21:40:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:40:19' 2026-02-20T21:40:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:40:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:40:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:40:19' 2026-02-20T21:40:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:40:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:40:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:40:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='OKOKOK142114264mirror.0trial10720.2.0-677-gf7870454trueOK42154266mirror.1trial10720.2.0-677-gf7870454falseOK42194269mirror.2trial10720.2.0-677-gf7870454falseOK42234272mirror.3trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":268285440.0,"last_snapshot_bytes":536570880,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623574,"remote_snapshot_timestamp":1771623574,"replay_state":"idle"}42114264mirror.0trial1072026-02-20 21:40:19cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911dup+stoppedlocal image is primary2026-02-20 21:40:19' 2026-02-20T21:40:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:40:37.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:40:37.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:40:19' 2026-02-20T21:40:37.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:40:37.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:40:37.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:40:37.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:40:37.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:40:37.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap3 2026-02-20T21:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap3 2026-02-20T21:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap3 2026-02-20T21:40:37.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:41:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:41:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:41:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:41:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:41:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:41:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:41:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:41:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:41:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:41:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:41:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:41:09.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:41:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:41:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:41:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:41:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:41:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:41:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:41:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:41:09.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:41:09.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:41:09.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:41:09.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:41:09.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:41:09.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:41:09.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:41:09.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:41:09.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:41:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:41:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:41:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:41:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:41:09.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:41:09.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:41:09.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:41:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:41:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:41:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:41:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:41:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:09.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:41:11.359 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 11 2026-02-20T21:41:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:41:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:41:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:41:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:41:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:41:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:41:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:41:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:41:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:41:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:41:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:41:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:41:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:41:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:41:11.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:41:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:41:11.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=11 2026-02-20T21:41:11.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:41:11.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:41:11.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:41:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:11.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:41:11.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:41:11.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:41:11.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 11 = '' 2026-02-20T21:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:41:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:41:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:12.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:12.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:12.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:12.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:41:12.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:41:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:41:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:41:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:41:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:41:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 11 = '' 2026-02-20T21:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:41:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:41:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:41:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:41:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:41:13.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:41:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:41:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:41:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:41:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:41:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 11 = '' 2026-02-20T21:41:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:41:13.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:41:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:41:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:41:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=11 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 11 = 11 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap3 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:41:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:41:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2026-02-20T21:41:16.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 6 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:41:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:40:24 2026 2026-02-20T21:41:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:41:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:41:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:41:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:41:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:41:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:41:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:41:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snap2-clean ']' 2026-02-20T21:41:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-20T21:41:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap2-clean 2026-02-20T21:41:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:41:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:41:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:16.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:41:16.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:41:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:41:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:41:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap3 2026-02-20T21:41:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap3 2026-02-20T21:41:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:41:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:41:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:41:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:41:17.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:41:18.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:41:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:41:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap3-clean 2026-02-20T21:41:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap3-clean 2026-02-20T21:41:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap3-clean 2026-02-20T21:41:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:41:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:41:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:41:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:41:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:41:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:41:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:41:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 57740288 bytes 2026-02-20T21:41:19.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 65866752 bytes 2026-02-20T21:41:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 137263104 bytes 2026-02-20T21:41:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 163284992 bytes 2026-02-20T21:41:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 170378240 bytes 2026-02-20T21:41:19.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 227581952 bytes 2026-02-20T21:41:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 246012928 bytes 2026-02-20T21:41:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 249773056 bytes 2026-02-20T21:41:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 296267776 bytes 2026-02-20T21:41:20.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 352755712 bytes 2026-02-20T21:41:20.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 353364992 bytes 2026-02-20T21:41:20.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 353983488 bytes 2026-02-20T21:41:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 436498432 bytes 2026-02-20T21:41:20.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 455124992 bytes 2026-02-20T21:41:20.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 518492160 bytes 2026-02-20T21:41:20.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9085478064438037) > weight (0.25) 2026-02-20T21:41:20.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.5616736017487796) > weight (0.25) 2026-02-20T21:41:20.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.38703037972053456) > weight (0.25) 2026-02-20T21:41:20.301 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7589934068058334) > weight (0.25) 2026-02-20T21:41:20.301 INFO:tasks.rbd_mirror_thrash:waiting for 64.67837678593293 secs before thrashing 2026-02-20T21:41:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 535170048 bytes 2026-02-20T21:41:20.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 582175744 bytes 2026-02-20T21:41:20.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 650601472 bytes 2026-02-20T21:41:20.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 715555840 bytes 2026-02-20T21:41:20.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 755518464 bytes 2026-02-20T21:41:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 782454784 bytes 2026-02-20T21:41:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 813471744 bytes 2026-02-20T21:41:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 864249856 bytes 2026-02-20T21:41:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 904215552 bytes 2026-02-20T21:41:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 931131392 bytes 2026-02-20T21:41:20.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 931744768 bytes 2026-02-20T21:41:20.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 947444736 bytes 2026-02-20T21:41:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 991833088 bytes 2026-02-20T21:41:20.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1102237696 bytes 2026-02-20T21:41:20.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1136646144 bytes 2026-02-20T21:41:20.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1163741184 bytes 2026-02-20T21:41:20.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1176557568 bytes 2026-02-20T21:41:20.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1225073664 bytes 2026-02-20T21:41:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1256731648 bytes 2026-02-20T21:41:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1323127808 bytes 2026-02-20T21:41:21.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1368300544 bytes 2026-02-20T21:41:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1412503552 bytes 2026-02-20T21:41:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1439228928 bytes 2026-02-20T21:41:21.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1526457344 bytes 2026-02-20T21:41:21.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1573100544 bytes 2026-02-20T21:41:21.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1595953152 bytes 2026-02-20T21:41:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1659782144 bytes 2026-02-20T21:41:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1729146880 bytes 2026-02-20T21:41:21.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1783932928 bytes 2026-02-20T21:41:21.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1873078272 bytes 2026-02-20T21:41:21.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1926604800 bytes 2026-02-20T21:41:21.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1927224320 bytes 2026-02-20T21:41:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1927838720 bytes 2026-02-20T21:41:21.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1975184384 bytes 2026-02-20T21:41:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2010973184 bytes 2026-02-20T21:41:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2070905856 bytes 2026-02-20T21:41:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2115008512 bytes 2026-02-20T21:41:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2164371456 bytes 2026-02-20T21:41:21.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2203844608 bytes 2026-02-20T21:41:21.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2296987648 bytes 2026-02-20T21:41:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2331694080 bytes 2026-02-20T21:41:21.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2405940224 bytes 2026-02-20T21:41:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2415752192 bytes 2026-02-20T21:41:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2458615808 bytes 2026-02-20T21:41:22.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2491557888 bytes 2026-02-20T21:41:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2534444032 bytes 2026-02-20T21:41:22.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2571450368 bytes 2026-02-20T21:41:22.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2592004096 bytes 2026-02-20T21:41:22.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2657964032 bytes 2026-02-20T21:41:22.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2702134272 bytes 2026-02-20T21:41:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2739390464 bytes 2026-02-20T21:41:22.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2790573056 bytes 2026-02-20T21:41:22.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2838802432 bytes 2026-02-20T21:41:22.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2839403520 bytes 2026-02-20T21:41:22.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2956881920 bytes 2026-02-20T21:41:22.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3008098304 bytes 2026-02-20T21:41:22.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3051923456 bytes 2026-02-20T21:41:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3116580864 bytes 2026-02-20T21:41:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3225791488 bytes 2026-02-20T21:41:22.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3277693952 bytes 2026-02-20T21:41:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3349886976 bytes 2026-02-20T21:41:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3395048448 bytes 2026-02-20T21:41:22.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3472330752 bytes 2026-02-20T21:41:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3476800512 bytes 2026-02-20T21:41:22.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3505514496 bytes 2026-02-20T21:41:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3567136768 bytes 2026-02-20T21:41:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3573521408 bytes 2026-02-20T21:41:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3647005696 bytes 2026-02-20T21:41:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3669797888 bytes 2026-02-20T21:41:23.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3702605824 bytes 2026-02-20T21:41:23.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3728098304 bytes 2026-02-20T21:41:23.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3780183040 bytes 2026-02-20T21:41:23.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3845168128 bytes 2026-02-20T21:41:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3860118528 bytes 2026-02-20T21:41:23.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3887447040 bytes 2026-02-20T21:41:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3942637568 bytes 2026-02-20T21:41:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4031327232 bytes 2026-02-20T21:41:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4080362496 bytes 2026-02-20T21:41:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4105596928 bytes 2026-02-20T21:41:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4167904256 bytes 2026-02-20T21:41:23.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4237598720 bytes 2026-02-20T21:41:23.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4293235712 bytes 2026-02-20T21:41:23.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4386985984 bytes 2026-02-20T21:41:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4401756160 bytes 2026-02-20T21:41:23.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4450141184 bytes 2026-02-20T21:41:23.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4484726784 bytes 2026-02-20T21:41:23.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4534311936 bytes 2026-02-20T21:41:23.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4588952576 bytes 2026-02-20T21:41:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4600330240 bytes 2026-02-20T21:41:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4628401152 bytes 2026-02-20T21:41:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4629729280 bytes 2026-02-20T21:41:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4643671040 bytes 2026-02-20T21:41:23.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4758070272 bytes 2026-02-20T21:41:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4793561088 bytes 2026-02-20T21:41:24.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4794169344 bytes 2026-02-20T21:41:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4794801152 bytes 2026-02-20T21:41:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4854804480 bytes 2026-02-20T21:41:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4894609408 bytes 2026-02-20T21:41:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4921672704 bytes 2026-02-20T21:41:24.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4970801152 bytes 2026-02-20T21:41:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5030592512 bytes 2026-02-20T21:41:24.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5058572288 bytes 2026-02-20T21:41:24.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5152427008 bytes 2026-02-20T21:41:24.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5172860928 bytes 2026-02-20T21:41:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5211710464 bytes 2026-02-20T21:41:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5249263616 bytes 2026-02-20T21:41:24.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5259417600 bytes 2026-02-20T21:41:24.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5270731776 bytes 2026-02-20T21:41:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5335090176 bytes 2026-02-20T21:41:24.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5372295168 bytes 2026-02-20T21:41:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5411073024 bytes 2026-02-20T21:41:24.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5415775232 bytes 2026-02-20T21:41:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5432979456 bytes 2026-02-20T21:41:24.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5448427520 bytes 2026-02-20T21:41:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5471370240 bytes 2026-02-20T21:41:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5484760064 bytes 2026-02-20T21:41:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5489951744 bytes 2026-02-20T21:41:24.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5545666560 bytes 2026-02-20T21:41:24.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5572150272 bytes 2026-02-20T21:41:24.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5594606592 bytes 2026-02-20T21:41:24.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5671122944 bytes 2026-02-20T21:41:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5697917952 bytes 2026-02-20T21:41:24.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5730168832 bytes 2026-02-20T21:41:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5784038400 bytes 2026-02-20T21:41:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5792978944 bytes 2026-02-20T21:41:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5823555584 bytes 2026-02-20T21:41:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5869460480 bytes 2026-02-20T21:41:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5918534656 bytes 2026-02-20T21:41:25.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5969479680 bytes 2026-02-20T21:41:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6053649408 bytes 2026-02-20T21:41:25.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6108015616 bytes 2026-02-20T21:41:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6207373312 bytes 2026-02-20T21:41:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6211454976 bytes 2026-02-20T21:41:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6263937024 bytes 2026-02-20T21:41:25.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6316269568 bytes 2026-02-20T21:41:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6341126144 bytes 2026-02-20T21:41:25.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6371788800 bytes 2026-02-20T21:41:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6398518272 bytes 2026-02-20T21:41:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:41:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:41:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:41:26.459 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.495313522484202) > weight (0.25) 2026-02-20T21:41:26.459 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7533290178582848) > weight (0.25) 2026-02-20T21:41:26.459 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-20T21:41:26.460 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-20T21:41:26.460 INFO:tasks.rbd_mirror_thrash:waiting for 9.595574038612689 secs before reviving daemons 2026-02-20T21:41:26.461 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T21:41:26.456+0000 7fad19905640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 19363) UID: 1000 2026-02-20T21:41:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:41:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:41:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:41:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:41:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:41:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:41:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:41:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:41:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":17871803.73,"bytes_per_snapshot":536276480.0,"last_snapshot_bytes":536154112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623670,"remote_snapshot_timestamp":1771623670,"replay_state":"idle"} 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.0 on trial107 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:41:19 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+stopped 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: local image is primary 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:41:19' 2026-02-20T21:41:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:41:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:41:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:41:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":17871803.73,"bytes_per_snapshot":536276480.0,"last_snapshot_bytes":536154112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623670,"remote_snapshot_timestamp":1771623670,"replay_state":"idle"} 2026-02-20T21:41:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:41:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:41:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:41:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:41:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.0 on trial107 2026-02-20T21:41:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:41:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:41:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:41:19' 2026-02-20T21:41:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:41:19' 2026-02-20T21:41:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:41:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:41:19' 2026-02-20T21:41:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:41:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:41:26.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:41:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='UNKNOWNUNKNOWNOK142114264mirror.0trial10720.2.0-677-gf7870454trueOK42154266mirror.1trial10720.2.0-677-gf7870454falseOK4219mirror.2trial10720.2.0-677-gf7870454falseUNKNOWN42234272mirror.3trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":17871803.73,"bytes_per_snapshot":536276480.0,"last_snapshot_bytes":536154112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623670,"remote_snapshot_timestamp":1771623670,"replay_state":"idle"}42114264mirror.0trial1072026-02-20 21:41:19cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911dup+stoppedlocal image is primary2026-02-20 21:41:19' 2026-02-20T21:41:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:41:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:41:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:41:19' 2026-02-20T21:41:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:41:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:41:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:41:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:41:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:41:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:41:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:41:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap4 2026-02-20T21:41:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap4 2026-02-20T21:41:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap4 2026-02-20T21:41:26.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:41:36.057 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-20T21:41:36.057 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:41:36.057 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-20T21:41:36.057 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-20T21:41:36.058 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-20T21:41:36.060 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-20T21:41:36.060 INFO:tasks.rbd_mirror_thrash:waiting for 83.89927499744971 secs before thrashing 2026-02-20T21:41:56.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:41:56.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:56.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:56.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:56.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:41:56.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:56.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:56.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:56.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:41:56.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:41:56.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:41:56.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:41:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:41:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:41:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:41:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:41:57.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:41:57.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:41:57.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:41:57.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:41:57.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:41:57.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:41:57.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:41:57.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:41:57.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:41:57.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:41:57.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:41:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:41:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:41:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:41:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:41:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:41:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:41:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:41:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:41:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:41:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:41:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:41:57.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:41:57.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:41:57.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:41:57.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:41:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:41:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:41:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:41:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:41:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:41:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:41:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:41:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:41:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:41:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:41:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:41:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:41:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:41:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:41:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:41:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 14 2026-02-20T21:41:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:41:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:41:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:41:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:41:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:41:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:41:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:41:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:41:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:41:59.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:41:59.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:41:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:41:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:41:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:41:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:41:59.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:41:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:41:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=14 2026-02-20T21:41:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:41:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:41:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:41:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:41:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:41:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:41:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:41:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:41:59.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:41:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:41:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:41:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:41:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:41:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:41:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:41:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 14 = '' 2026-02-20T21:41:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:41:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:42:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:00.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:42:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:42:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:42:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:42:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:42:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:42:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:42:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:42:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 14 = '' 2026-02-20T21:42:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:42:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:42:01.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:01.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:01.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:01.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:01.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:01.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:42:01.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:42:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:42:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:42:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:42:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:42:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:42:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:42:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 14 = '' 2026-02-20T21:42:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:42:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:42:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:42:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:42:02.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=14 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 14 = 14 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap4 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:42:02.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:42:03.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2026-02-20T21:42:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:42:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:42:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:42:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 8 2026-02-20T21:42:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:42:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:42:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:42:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:42:03.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:42:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:42:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:42:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:40:24 2026 2026-02-20T21:42:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:42:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:42:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:42:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:42:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:42:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:42:03.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snap3-clean ']' 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap3-clean 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:42:04.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:42:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:42:05.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:05.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap4 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap4 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:05.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:42:05.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:42:05.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:42:06.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap4-clean 2026-02-20T21:42:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap4-clean 2026-02-20T21:42:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:42:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap4-clean 2026-02-20T21:42:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:42:07.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:42:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:42:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:42:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:07.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:42:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=11 2026-02-20T21:42:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:42:07.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:42:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 25720832 bytes 2026-02-20T21:42:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 61421568 bytes 2026-02-20T21:42:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 111238144 bytes 2026-02-20T21:42:07.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 130635776 bytes 2026-02-20T21:42:07.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 240848896 bytes 2026-02-20T21:42:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 292149248 bytes 2026-02-20T21:42:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 369767424 bytes 2026-02-20T21:42:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 436291584 bytes 2026-02-20T21:42:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 445997056 bytes 2026-02-20T21:42:08.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 525664256 bytes 2026-02-20T21:42:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 559891456 bytes 2026-02-20T21:42:08.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 614866944 bytes 2026-02-20T21:42:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 639793152 bytes 2026-02-20T21:42:08.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 709777408 bytes 2026-02-20T21:42:08.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 752178176 bytes 2026-02-20T21:42:08.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 789830656 bytes 2026-02-20T21:42:08.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 842670080 bytes 2026-02-20T21:42:08.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 889277440 bytes 2026-02-20T21:42:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 924093440 bytes 2026-02-20T21:42:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 927040512 bytes 2026-02-20T21:42:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 938174464 bytes 2026-02-20T21:42:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 961147904 bytes 2026-02-20T21:42:08.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 996156416 bytes 2026-02-20T21:42:08.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1025460224 bytes 2026-02-20T21:42:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1026072576 bytes 2026-02-20T21:42:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1067747328 bytes 2026-02-20T21:42:08.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1084925952 bytes 2026-02-20T21:42:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1164493824 bytes 2026-02-20T21:42:08.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1199279104 bytes 2026-02-20T21:42:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1222877184 bytes 2026-02-20T21:42:09.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1339390976 bytes 2026-02-20T21:42:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1434115072 bytes 2026-02-20T21:42:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1461251072 bytes 2026-02-20T21:42:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1534647296 bytes 2026-02-20T21:42:09.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1578703872 bytes 2026-02-20T21:42:09.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1618029568 bytes 2026-02-20T21:42:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1646241792 bytes 2026-02-20T21:42:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1698672640 bytes 2026-02-20T21:42:09.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1750018048 bytes 2026-02-20T21:42:09.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1757737984 bytes 2026-02-20T21:42:09.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1804493824 bytes 2026-02-20T21:42:09.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1872082944 bytes 2026-02-20T21:42:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1872694272 bytes 2026-02-20T21:42:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1971694592 bytes 2026-02-20T21:42:09.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2020592640 bytes 2026-02-20T21:42:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2080946176 bytes 2026-02-20T21:42:09.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2103453696 bytes 2026-02-20T21:42:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2131692544 bytes 2026-02-20T21:42:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2164474880 bytes 2026-02-20T21:42:09.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2229991424 bytes 2026-02-20T21:42:09.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2230594560 bytes 2026-02-20T21:42:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2288675840 bytes 2026-02-20T21:42:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2331225088 bytes 2026-02-20T21:42:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2331814912 bytes 2026-02-20T21:42:10.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2408524800 bytes 2026-02-20T21:42:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2451497984 bytes 2026-02-20T21:42:10.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2465583104 bytes 2026-02-20T21:42:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2501394432 bytes 2026-02-20T21:42:10.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2562448384 bytes 2026-02-20T21:42:10.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2600828928 bytes 2026-02-20T21:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2611701760 bytes 2026-02-20T21:42:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2637332480 bytes 2026-02-20T21:42:10.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2642066432 bytes 2026-02-20T21:42:10.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2689567744 bytes 2026-02-20T21:42:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2726530048 bytes 2026-02-20T21:42:10.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2762179584 bytes 2026-02-20T21:42:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2846436352 bytes 2026-02-20T21:42:10.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2856273920 bytes 2026-02-20T21:42:10.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2876520448 bytes 2026-02-20T21:42:10.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2877580288 bytes 2026-02-20T21:42:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2910440448 bytes 2026-02-20T21:42:10.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2934125568 bytes 2026-02-20T21:42:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2980047872 bytes 2026-02-20T21:42:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3042051072 bytes 2026-02-20T21:42:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3065305088 bytes 2026-02-20T21:42:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3133770752 bytes 2026-02-20T21:42:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3222732800 bytes 2026-02-20T21:42:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3223360512 bytes 2026-02-20T21:42:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3252069376 bytes 2026-02-20T21:42:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3265342464 bytes 2026-02-20T21:42:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3341608960 bytes 2026-02-20T21:42:11.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3406334976 bytes 2026-02-20T21:42:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3482248192 bytes 2026-02-20T21:42:11.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3517904896 bytes 2026-02-20T21:42:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3553623040 bytes 2026-02-20T21:42:11.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3632844800 bytes 2026-02-20T21:42:11.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3651886080 bytes 2026-02-20T21:42:11.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3707772928 bytes 2026-02-20T21:42:11.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3733223424 bytes 2026-02-20T21:42:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3766124544 bytes 2026-02-20T21:42:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3798065152 bytes 2026-02-20T21:42:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3841847296 bytes 2026-02-20T21:42:11.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3886730240 bytes 2026-02-20T21:42:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3942057984 bytes 2026-02-20T21:42:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4131346432 bytes 2026-02-20T21:42:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4195101696 bytes 2026-02-20T21:42:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4203361280 bytes 2026-02-20T21:42:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4234728448 bytes 2026-02-20T21:42:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4290302976 bytes 2026-02-20T21:42:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4319038464 bytes 2026-02-20T21:42:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4398997504 bytes 2026-02-20T21:42:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4467462144 bytes 2026-02-20T21:42:12.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4558417920 bytes 2026-02-20T21:42:12.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4592389120 bytes 2026-02-20T21:42:12.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4656811008 bytes 2026-02-20T21:42:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4686034944 bytes 2026-02-20T21:42:12.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4710214656 bytes 2026-02-20T21:42:12.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4710828032 bytes 2026-02-20T21:42:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4734241792 bytes 2026-02-20T21:42:12.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4749248512 bytes 2026-02-20T21:42:12.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4843919360 bytes 2026-02-20T21:42:12.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4916177920 bytes 2026-02-20T21:42:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4933534720 bytes 2026-02-20T21:42:12.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5028582400 bytes 2026-02-20T21:42:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5056604160 bytes 2026-02-20T21:42:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5095388160 bytes 2026-02-20T21:42:12.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5103667200 bytes 2026-02-20T21:42:12.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5114358784 bytes 2026-02-20T21:42:12.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5154419712 bytes 2026-02-20T21:42:12.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5214235648 bytes 2026-02-20T21:42:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5247918080 bytes 2026-02-20T21:42:12.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5299097600 bytes 2026-02-20T21:42:12.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5334141952 bytes 2026-02-20T21:42:12.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5366815744 bytes 2026-02-20T21:42:12.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5421974528 bytes 2026-02-20T21:42:12.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5468421120 bytes 2026-02-20T21:42:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5507189760 bytes 2026-02-20T21:42:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5588464640 bytes 2026-02-20T21:42:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5589825536 bytes 2026-02-20T21:42:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5595143168 bytes 2026-02-20T21:42:13.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5623262208 bytes 2026-02-20T21:42:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5686056960 bytes 2026-02-20T21:42:13.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5738597376 bytes 2026-02-20T21:42:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5813029888 bytes 2026-02-20T21:42:13.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5840916480 bytes 2026-02-20T21:42:13.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5898100736 bytes 2026-02-20T21:42:13.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5974013952 bytes 2026-02-20T21:42:13.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5997261824 bytes 2026-02-20T21:42:13.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6061872128 bytes 2026-02-20T21:42:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6093602816 bytes 2026-02-20T21:42:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6112968704 bytes 2026-02-20T21:42:13.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6140555264 bytes 2026-02-20T21:42:13.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6181432320 bytes 2026-02-20T21:42:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6251224064 bytes 2026-02-20T21:42:13.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6283538432 bytes 2026-02-20T21:42:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6324797440 bytes 2026-02-20T21:42:13.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6410437632 bytes 2026-02-20T21:42:13.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6452508672 bytes 2026-02-20T21:42:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6453111808 bytes 2026-02-20T21:42:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6495339520 bytes 2026-02-20T21:42:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6518919168 bytes 2026-02-20T21:42:13.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6565052416 bytes 2026-02-20T21:42:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6565683200 bytes 2026-02-20T21:42:14.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6628755456 bytes 2026-02-20T21:42:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6712752128 bytes 2026-02-20T21:42:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6755725312 bytes 2026-02-20T21:42:14.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6774864896 bytes 2026-02-20T21:42:14.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6847491072 bytes 2026-02-20T21:42:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6891722752 bytes 2026-02-20T21:42:14.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6940604416 bytes 2026-02-20T21:42:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7030016000 bytes 2026-02-20T21:42:14.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7057091584 bytes 2026-02-20T21:42:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7095088128 bytes 2026-02-20T21:42:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7112042496 bytes 2026-02-20T21:42:14.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7145127936 bytes 2026-02-20T21:42:14.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7174248448 bytes 2026-02-20T21:42:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7199313920 bytes 2026-02-20T21:42:14.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7222936576 bytes 2026-02-20T21:42:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7223538688 bytes 2026-02-20T21:42:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7243302912 bytes 2026-02-20T21:42:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7248759808 bytes 2026-02-20T21:42:14.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7274085376 bytes 2026-02-20T21:42:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7320606720 bytes 2026-02-20T21:42:14.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7351573504 bytes 2026-02-20T21:42:14.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7370368000 bytes 2026-02-20T21:42:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7419273216 bytes 2026-02-20T21:42:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7468454912 bytes 2026-02-20T21:42:14.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7524034560 bytes 2026-02-20T21:42:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7555195904 bytes 2026-02-20T21:42:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7586703360 bytes 2026-02-20T21:42:15.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7618064384 bytes 2026-02-20T21:42:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7668536320 bytes 2026-02-20T21:42:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7676093440 bytes 2026-02-20T21:42:15.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7725196288 bytes 2026-02-20T21:42:15.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7838547968 bytes 2026-02-20T21:42:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7855994880 bytes 2026-02-20T21:42:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7856585728 bytes 2026-02-20T21:42:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7907714048 bytes 2026-02-20T21:42:15.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7940315136 bytes 2026-02-20T21:42:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7976835072 bytes 2026-02-20T21:42:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8024699904 bytes 2026-02-20T21:42:15.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8072306688 bytes 2026-02-20T21:42:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8091564032 bytes 2026-02-20T21:42:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8219632640 bytes 2026-02-20T21:42:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8220236800 bytes 2026-02-20T21:42:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8243145728 bytes 2026-02-20T21:42:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8299534336 bytes 2026-02-20T21:42:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8334067712 bytes 2026-02-20T21:42:15.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8393771008 bytes 2026-02-20T21:42:15.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8435111936 bytes 2026-02-20T21:42:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8528514048 bytes 2026-02-20T21:42:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8571414528 bytes 2026-02-20T21:42:16.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8633897984 bytes 2026-02-20T21:42:16.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8685316096 bytes 2026-02-20T21:42:16.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8723292160 bytes 2026-02-20T21:42:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8753720320 bytes 2026-02-20T21:42:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8808854528 bytes 2026-02-20T21:42:16.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8877734912 bytes 2026-02-20T21:42:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8912503808 bytes 2026-02-20T21:42:16.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8923684864 bytes 2026-02-20T21:42:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8973767680 bytes 2026-02-20T21:42:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9000066048 bytes 2026-02-20T21:42:16.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9098337280 bytes 2026-02-20T21:42:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9180511232 bytes 2026-02-20T21:42:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9202664448 bytes 2026-02-20T21:42:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9255285760 bytes 2026-02-20T21:42:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9259356160 bytes 2026-02-20T21:42:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9303454720 bytes 2026-02-20T21:42:16.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9350751232 bytes 2026-02-20T21:42:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9381683200 bytes 2026-02-20T21:42:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9426358272 bytes 2026-02-20T21:42:16.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9448890368 bytes 2026-02-20T21:42:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9494516736 bytes 2026-02-20T21:42:16.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9572305920 bytes 2026-02-20T21:42:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9593768960 bytes 2026-02-20T21:42:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9604682752 bytes 2026-02-20T21:42:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9612042240 bytes 2026-02-20T21:42:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9625142272 bytes 2026-02-20T21:42:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9669654528 bytes 2026-02-20T21:42:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9695970304 bytes 2026-02-20T21:42:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9769536512 bytes 2026-02-20T21:42:17.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9782739968 bytes 2026-02-20T21:42:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9814515712 bytes 2026-02-20T21:42:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9861918720 bytes 2026-02-20T21:42:17.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9862530048 bytes 2026-02-20T21:42:17.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 9913467904 bytes 2026-02-20T21:42:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 9955586048 bytes 2026-02-20T21:42:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 9975841792 bytes 2026-02-20T21:42:17.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 10011945984 bytes 2026-02-20T21:42:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 10065188864 bytes 2026-02-20T21:42:17.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 10109889536 bytes 2026-02-20T21:42:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10202798080 bytes 2026-02-20T21:42:17.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 10234004480 bytes 2026-02-20T21:42:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 10289713152 bytes 2026-02-20T21:42:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10335260672 bytes 2026-02-20T21:42:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10341365760 bytes 2026-02-20T21:42:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10395009024 bytes 2026-02-20T21:42:17.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10468667392 bytes 2026-02-20T21:42:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 10481036288 bytes 2026-02-20T21:42:17.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 10551801856 bytes 2026-02-20T21:42:17.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25100 IOs, 10583988224 bytes 2026-02-20T21:42:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25200 IOs, 10619376640 bytes 2026-02-20T21:42:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25300 IOs, 10678836224 bytes 2026-02-20T21:42:18.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25400 IOs, 10732774400 bytes 2026-02-20T21:42:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25500 IOs, 10824732672 bytes 2026-02-20T21:42:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25600 IOs, 10831779840 bytes 2026-02-20T21:42:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25700 IOs, 10913510400 bytes 2026-02-20T21:42:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25800 IOs, 10967402496 bytes 2026-02-20T21:42:18.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25900 IOs, 11015723008 bytes 2026-02-20T21:42:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26000 IOs, 11040600064 bytes 2026-02-20T21:42:18.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26100 IOs, 11091061760 bytes 2026-02-20T21:42:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26200 IOs, 11104001024 bytes 2026-02-20T21:42:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26300 IOs, 11186843648 bytes 2026-02-20T21:42:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26400 IOs, 11214615552 bytes 2026-02-20T21:42:18.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26500 IOs, 11288973312 bytes 2026-02-20T21:42:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:42:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:42:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:42:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:42:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:42:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:42:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:42:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:42:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:42:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:42:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:42:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:42:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:42:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:42:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:42:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:42:19.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:42:19.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:42:19.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:42:19.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":17871803.73,"bytes_per_snapshot":536276480.0,"last_snapshot_bytes":536154112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623670,"remote_snapshot_timestamp":1771623670,"replay_state":"idle"} 2026-02-20T21:42:19.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.0 on trial107 2026-02-20T21:42:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:41:49 2026-02-20T21:42:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:42:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:42:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+stopped 2026-02-20T21:42:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: local image is primary 2026-02-20T21:42:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:41:49' 2026-02-20T21:42:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:42:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:42:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:42:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":17871803.73,"bytes_per_snapshot":536276480.0,"last_snapshot_bytes":536154112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623670,"remote_snapshot_timestamp":1771623670,"replay_state":"idle"} 2026-02-20T21:42:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:42:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:42:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:42:19.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:42:19.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:42:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.0 on trial107 2026-02-20T21:42:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:42:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:42:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:41:49' 2026-02-20T21:42:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:41:49' 2026-02-20T21:42:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:42:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:41:49' 2026-02-20T21:42:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:42:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:42:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:42:19.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='OKOKOK142114264mirror.0trial10720.2.0-677-gf7870454trueOK42154266mirror.1trial10720.2.0-677-gf7870454falseOK42234272mirror.3trial10720.2.0-677-gf7870454falseOK43894391mirror.2trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":17871803.73,"bytes_per_snapshot":536276480.0,"last_snapshot_bytes":536154112,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623670,"remote_snapshot_timestamp":1771623670,"replay_state":"idle"}42114264mirror.0trial1072026-02-20 21:41:49cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911dup+stoppedlocal image is primary2026-02-20 21:41:49' 2026-02-20T21:42:19.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:42:19.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:42:19.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:41:49' 2026-02-20T21:42:19.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:42:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:42:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:42:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:42:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:42:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap5 2026-02-20T21:42:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap5 2026-02-20T21:42:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:42:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap5 2026-02-20T21:42:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:42:24.980 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:42:24.981 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:42:24.981 INFO:tasks.rbd_mirror_thrash:waiting for 5.877786494376047 secs before reviving daemons 2026-02-20T21:42:24.982 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T21:42:24.975+0000 7f0339544640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 19413) UID: 1000 2026-02-20T21:42:30.860 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:42:30.860 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:42:30.860 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:42:30.860 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:42:30.861 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:42:30.863 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:42:30.863 INFO:tasks.rbd_mirror_thrash:waiting for 86.63021099576532 secs before thrashing 2026-02-20T21:42:49.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:42:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:42:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:42:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:42:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:42:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:42:50.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:42:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:42:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:42:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:42:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:42:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:42:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:42:50.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:42:50.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:42:50.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:42:50.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:42:50.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:42:50.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:42:50.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:42:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:42:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:42:50.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:42:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:42:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:42:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:42:50.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:42:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:42:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:42:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:42:50.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:42:50.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:42:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:42:50.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:50.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:50.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:42:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 17 2026-02-20T21:42:52.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:42:52.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:42:52.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:42:52.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:42:52.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:42:52.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:52.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:42:52.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:42:52.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:42:52.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:42:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:42:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:42:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:42:52.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:52.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:42:52.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:42:52.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:52.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:52.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:42:52.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:42:52.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:42:52.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:42:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=17 2026-02-20T21:42:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:42:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:42:52.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:42:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:42:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:42:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:42:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:42:52.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:42:52.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:42:52.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:42:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:42:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 17 = '' 2026-02-20T21:42:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:42:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:42:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:42:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:42:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:42:53.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:42:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:42:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:42:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:42:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:42:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 17 = '' 2026-02-20T21:42:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:42:53.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:42:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:42:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:42:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:42:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:42:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=17 2026-02-20T21:42:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 17 = 17 2026-02-20T21:42:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2026-02-20T21:42:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:42:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap5 2026-02-20T21:42:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:42:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:42:54.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:42:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2026-02-20T21:42:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:42:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:42:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:42:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 10 2026-02-20T21:42:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:42:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:42:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:42:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:42:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:42:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:42:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:42:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:42:04 2026 2026-02-20T21:42:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:42:55.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:42:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:42:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:42:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:42:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:42:55.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snap4-clean ']' 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap4-clean 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:42:55.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:42:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:56.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:42:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap5 2026-02-20T21:42:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap5 2026-02-20T21:42:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:42:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:42:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:42:57.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:42:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:57.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:42:57.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:42:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:42:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap5-clean 2026-02-20T21:42:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap5-clean 2026-02-20T21:42:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:42:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap5-clean 2026-02-20T21:42:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:42:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:42:58.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:42:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:42:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:42:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:42:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:42:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T21:42:58.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:42:58.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:42:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 88291328 bytes 2026-02-20T21:42:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 164446208 bytes 2026-02-20T21:42:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 187286528 bytes 2026-02-20T21:42:59.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 230897664 bytes 2026-02-20T21:42:59.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 264197120 bytes 2026-02-20T21:42:59.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 287881216 bytes 2026-02-20T21:42:59.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 334540800 bytes 2026-02-20T21:42:59.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 345366528 bytes 2026-02-20T21:42:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 370337792 bytes 2026-02-20T21:42:59.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 451506176 bytes 2026-02-20T21:42:59.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 506962944 bytes 2026-02-20T21:42:59.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 557289472 bytes 2026-02-20T21:42:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 669076480 bytes 2026-02-20T21:42:59.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 739656704 bytes 2026-02-20T21:42:59.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 800309248 bytes 2026-02-20T21:42:59.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 855613440 bytes 2026-02-20T21:42:59.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 881303552 bytes 2026-02-20T21:42:59.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 947030016 bytes 2026-02-20T21:42:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 1004328960 bytes 2026-02-20T21:42:59.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1025521664 bytes 2026-02-20T21:42:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1040830464 bytes 2026-02-20T21:42:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1086780416 bytes 2026-02-20T21:42:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1157521408 bytes 2026-02-20T21:42:59.961 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-20T21:42:59.961 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-20T21:42:59.961 INFO:tasks.rbd_mirror_thrash:waiting for 8.597786114842293 secs before reviving daemons 2026-02-20T21:42:59.962 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T21:42:59.955+0000 7f5a307e2640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 19335) UID: 1000 2026-02-20T21:42:59.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1224434688 bytes 2026-02-20T21:43:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1235870720 bytes 2026-02-20T21:43:00.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1256724480 bytes 2026-02-20T21:43:00.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1276832768 bytes 2026-02-20T21:43:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1309660160 bytes 2026-02-20T21:43:00.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1391780864 bytes 2026-02-20T21:43:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1421491200 bytes 2026-02-20T21:43:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1445189632 bytes 2026-02-20T21:43:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1529364480 bytes 2026-02-20T21:43:00.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1591192576 bytes 2026-02-20T21:43:00.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1654289408 bytes 2026-02-20T21:43:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1654899712 bytes 2026-02-20T21:43:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1674576896 bytes 2026-02-20T21:43:00.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1696526336 bytes 2026-02-20T21:43:00.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1769563136 bytes 2026-02-20T21:43:00.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1855564800 bytes 2026-02-20T21:43:00.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1938545664 bytes 2026-02-20T21:43:00.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1967498240 bytes 2026-02-20T21:43:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2007259136 bytes 2026-02-20T21:43:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2007873536 bytes 2026-02-20T21:43:00.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2050940928 bytes 2026-02-20T21:43:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2094342144 bytes 2026-02-20T21:43:00.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2094971904 bytes 2026-02-20T21:43:00.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2144094208 bytes 2026-02-20T21:43:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2201784320 bytes 2026-02-20T21:43:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2236479488 bytes 2026-02-20T21:43:01.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2237109248 bytes 2026-02-20T21:43:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2306818048 bytes 2026-02-20T21:43:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2351780864 bytes 2026-02-20T21:43:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2397396992 bytes 2026-02-20T21:43:01.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2398021632 bytes 2026-02-20T21:43:01.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2433127424 bytes 2026-02-20T21:43:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2493800448 bytes 2026-02-20T21:43:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2503363584 bytes 2026-02-20T21:43:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2535857152 bytes 2026-02-20T21:43:01.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2558074880 bytes 2026-02-20T21:43:01.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2579723264 bytes 2026-02-20T21:43:01.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2627032064 bytes 2026-02-20T21:43:01.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2634776576 bytes 2026-02-20T21:43:01.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2675757056 bytes 2026-02-20T21:43:01.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2705803264 bytes 2026-02-20T21:43:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2733572096 bytes 2026-02-20T21:43:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2813814784 bytes 2026-02-20T21:43:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2871002112 bytes 2026-02-20T21:43:01.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2917423104 bytes 2026-02-20T21:43:02.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2951306240 bytes 2026-02-20T21:43:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2969356288 bytes 2026-02-20T21:43:02.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2985218048 bytes 2026-02-20T21:43:02.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3037672448 bytes 2026-02-20T21:43:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3086314496 bytes 2026-02-20T21:43:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3121513472 bytes 2026-02-20T21:43:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3153597440 bytes 2026-02-20T21:43:02.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3260509184 bytes 2026-02-20T21:43:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3283175424 bytes 2026-02-20T21:43:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3300470784 bytes 2026-02-20T21:43:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3344703488 bytes 2026-02-20T21:43:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3430933504 bytes 2026-02-20T21:43:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3515514880 bytes 2026-02-20T21:43:02.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3556383744 bytes 2026-02-20T21:43:02.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3575249920 bytes 2026-02-20T21:43:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3593380864 bytes 2026-02-20T21:43:02.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3622386688 bytes 2026-02-20T21:43:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3647708160 bytes 2026-02-20T21:43:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3705606144 bytes 2026-02-20T21:43:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3727168512 bytes 2026-02-20T21:43:02.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3782804480 bytes 2026-02-20T21:43:02.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3845654528 bytes 2026-02-20T21:43:02.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3872391168 bytes 2026-02-20T21:43:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3901574144 bytes 2026-02-20T21:43:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3944978432 bytes 2026-02-20T21:43:03.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3986422784 bytes 2026-02-20T21:43:03.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3989939200 bytes 2026-02-20T21:43:03.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4007171072 bytes 2026-02-20T21:43:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4042889216 bytes 2026-02-20T21:43:03.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4071998464 bytes 2026-02-20T21:43:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4072640512 bytes 2026-02-20T21:43:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4132811776 bytes 2026-02-20T21:43:03.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4189838336 bytes 2026-02-20T21:43:03.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4198780928 bytes 2026-02-20T21:43:03.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4265329664 bytes 2026-02-20T21:43:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4275685376 bytes 2026-02-20T21:43:03.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4298719232 bytes 2026-02-20T21:43:03.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4323679232 bytes 2026-02-20T21:43:03.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4354122752 bytes 2026-02-20T21:43:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4375712768 bytes 2026-02-20T21:43:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4407112704 bytes 2026-02-20T21:43:03.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4459700224 bytes 2026-02-20T21:43:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4521814016 bytes 2026-02-20T21:43:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4532724736 bytes 2026-02-20T21:43:03.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4559621120 bytes 2026-02-20T21:43:03.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4632918016 bytes 2026-02-20T21:43:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4633529344 bytes 2026-02-20T21:43:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4707935232 bytes 2026-02-20T21:43:03.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4749968384 bytes 2026-02-20T21:43:04.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4802598912 bytes 2026-02-20T21:43:04.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4877935616 bytes 2026-02-20T21:43:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4903392256 bytes 2026-02-20T21:43:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4922372096 bytes 2026-02-20T21:43:04.207 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4980279296 bytes 2026-02-20T21:43:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4992143360 bytes 2026-02-20T21:43:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5013610496 bytes 2026-02-20T21:43:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5039899648 bytes 2026-02-20T21:43:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5061286912 bytes 2026-02-20T21:43:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5101599744 bytes 2026-02-20T21:43:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5151491072 bytes 2026-02-20T21:43:04.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5154486272 bytes 2026-02-20T21:43:04.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5238362112 bytes 2026-02-20T21:43:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5287353344 bytes 2026-02-20T21:43:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5327083520 bytes 2026-02-20T21:43:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5337302016 bytes 2026-02-20T21:43:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5404665856 bytes 2026-02-20T21:43:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5405283328 bytes 2026-02-20T21:43:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5424335872 bytes 2026-02-20T21:43:04.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5457839104 bytes 2026-02-20T21:43:04.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5494116352 bytes 2026-02-20T21:43:04.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5499753472 bytes 2026-02-20T21:43:04.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5500385280 bytes 2026-02-20T21:43:04.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5518349312 bytes 2026-02-20T21:43:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5562659840 bytes 2026-02-20T21:43:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5590686720 bytes 2026-02-20T21:43:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5603861504 bytes 2026-02-20T21:43:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5643670528 bytes 2026-02-20T21:43:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5707930624 bytes 2026-02-20T21:43:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5769843712 bytes 2026-02-20T21:43:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5805091840 bytes 2026-02-20T21:43:05.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5843532800 bytes 2026-02-20T21:43:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5858327552 bytes 2026-02-20T21:43:05.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 5887232000 bytes 2026-02-20T21:43:05.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 5981884416 bytes 2026-02-20T21:43:05.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6044386304 bytes 2026-02-20T21:43:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6150890496 bytes 2026-02-20T21:43:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6196527104 bytes 2026-02-20T21:43:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6249013248 bytes 2026-02-20T21:43:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6296741888 bytes 2026-02-20T21:43:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6324507648 bytes 2026-02-20T21:43:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6359312384 bytes 2026-02-20T21:43:05.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6406242304 bytes 2026-02-20T21:43:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6445868032 bytes 2026-02-20T21:43:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6474482688 bytes 2026-02-20T21:43:05.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6517434368 bytes 2026-02-20T21:43:05.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6565454848 bytes 2026-02-20T21:43:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6571874304 bytes 2026-02-20T21:43:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6702516224 bytes 2026-02-20T21:43:06.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6755936256 bytes 2026-02-20T21:43:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6813611008 bytes 2026-02-20T21:43:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6894836736 bytes 2026-02-20T21:43:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6938579968 bytes 2026-02-20T21:43:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6989721600 bytes 2026-02-20T21:43:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7054225408 bytes 2026-02-20T21:43:06.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7080078336 bytes 2026-02-20T21:43:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7097436160 bytes 2026-02-20T21:43:06.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7155415040 bytes 2026-02-20T21:43:06.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7188461568 bytes 2026-02-20T21:43:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7248022528 bytes 2026-02-20T21:43:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7368668160 bytes 2026-02-20T21:43:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7426526208 bytes 2026-02-20T21:43:06.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7470840832 bytes 2026-02-20T21:43:06.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7548551168 bytes 2026-02-20T21:43:06.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7575783424 bytes 2026-02-20T21:43:06.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7685317632 bytes 2026-02-20T21:43:06.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7767163904 bytes 2026-02-20T21:43:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7793607680 bytes 2026-02-20T21:43:07.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7810972672 bytes 2026-02-20T21:43:07.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7860385792 bytes 2026-02-20T21:43:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7875858432 bytes 2026-02-20T21:43:07.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7916950528 bytes 2026-02-20T21:43:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7981553664 bytes 2026-02-20T21:43:07.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7997586432 bytes 2026-02-20T21:43:07.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8031071232 bytes 2026-02-20T21:43:07.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8080650240 bytes 2026-02-20T21:43:07.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8122287104 bytes 2026-02-20T21:43:07.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8153307136 bytes 2026-02-20T21:43:07.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8169773056 bytes 2026-02-20T21:43:07.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8226390016 bytes 2026-02-20T21:43:07.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8278888448 bytes 2026-02-20T21:43:07.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8312560640 bytes 2026-02-20T21:43:07.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8349657088 bytes 2026-02-20T21:43:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8405033984 bytes 2026-02-20T21:43:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:43:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:43:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:43:07.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:43:08.560 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-20T21:43:08.560 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:43:08.561 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-20T21:43:08.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T21:43:08.561 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T21:43:08.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T21:43:08.563 INFO:tasks.rbd_mirror_thrash:waiting for 78.66639775902094 secs before thrashing 2026-02-20T21:43:08.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:43:08.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:43:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:43:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:43:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:43:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:43:08.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:43:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:43:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:43:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:43:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623771,"remote_snapshot_timestamp":1771623771,"replay_state":"idle"} 2026-02-20T21:43:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.3 on trial107 2026-02-20T21:43:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:43:04 2026-02-20T21:43:08.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:43:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:43:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+stopped 2026-02-20T21:43:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: local image is primary 2026-02-20T21:43:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:43:01' 2026-02-20T21:43:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:43:08.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:43:08.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:43:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623771,"remote_snapshot_timestamp":1771623771,"replay_state":"idle"} 2026-02-20T21:43:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:43:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:43:08.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:43:08.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:43:08.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:43:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.3 on trial107 2026-02-20T21:43:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:43:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:43:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:43:04' 2026-02-20T21:43:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:43:04' 2026-02-20T21:43:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:43:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:43:04' 2026-02-20T21:43:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:43:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:43:08.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:43:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='OKOKOK142154266mirror.1trial10720.2.0-677-gf7870454falseOK42234272mirror.3trial10720.2.0-677-gf7870454trueOK43894391mirror.2trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623771,"remote_snapshot_timestamp":1771623771,"replay_state":"idle"}42234272mirror.3trial1072026-02-20 21:43:04cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911dup+stoppedlocal image is primary2026-02-20 21:43:01' 2026-02-20T21:43:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:43:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:43:08.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:43:04' 2026-02-20T21:43:08.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:43:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:43:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:43:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:43:08.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:43:08.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:43:08.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:43:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap6 2026-02-20T21:43:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap6 2026-02-20T21:43:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:43:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap6 2026-02-20T21:43:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:43:39.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:43:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:43:40.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:43:40.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:40.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:40.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:40.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:43:40.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:43:40.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:43:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:43:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:43:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:43:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:43:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:43:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:43:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:43:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:43:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:43:40.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:43:40.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:43:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:43:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:43:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:43:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:43:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:43:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:43:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:43:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:43:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:43:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:43:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:43:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:43:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:43:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:43:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:43:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:43:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:43:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:43:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:43:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:43:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:43:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:43:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:43:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:43:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:43:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:43:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:43:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:43:40.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:43:40.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:43:40.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:43:40.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:43:40.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:43:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:43:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:43:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:43:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:43:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:43:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:43:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:43:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:43:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:43:40.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:43:40.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:43:40.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:43:40.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:43:40.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:43:40.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:43:40.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:43:40.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:43:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:43:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:43:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:43:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:43:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:43:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:43:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:43:40.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:43:40.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:43:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:43:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:43:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:43:40.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:43:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:43:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:43:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:43:40.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:43:40.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:43:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:43:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:43:40.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:43:40.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:43:40.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:43:40.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:43:41.402 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:43:41.395+0000 7fa5e2658640 -1 reset not still connected to 0x55b901b3fad0 2026-02-20T21:43:42.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 20 2026-02-20T21:43:42.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:43:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:43:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:43:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:43:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:43:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:43:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:43:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:43:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:43:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:43:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:43:42.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:43:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:43:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:43:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:43:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:43:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:43:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:43:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:43:42.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=20 2026-02-20T21:43:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:43:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:43:42.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:43:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:43:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:43:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:43:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:43:42.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:43:42.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:43:42.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:43:42.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:43:42.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:43:42.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:43:42.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 20 = '' 2026-02-20T21:43:42.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:43:42.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:43:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:43:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:43:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:43:43.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:43:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:43:43.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:43:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:43:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:43:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:43:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:43:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 20 = '' 2026-02-20T21:43:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:43:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:43:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:43:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:43:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:43:43.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:43:43.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:43:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=20 2026-02-20T21:43:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 20 = 20 2026-02-20T21:43:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:43:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2026-02-20T21:43:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:43.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap6 2026-02-20T21:43:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:43:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:43:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:43:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2026-02-20T21:43:44.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 12 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:43:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:42:04 2026 2026-02-20T21:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:43:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snap5-clean ']' 2026-02-20T21:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-20T21:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap5-clean 2026-02-20T21:43:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:43:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:43:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:43:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:43:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:43:46.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:43:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap6 2026-02-20T21:43:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap6 2026-02-20T21:43:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:43:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:43:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:46.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:43:46.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:43:46.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:47.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:43:47.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:47.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:43:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:43:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap6-clean 2026-02-20T21:43:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap6-clean 2026-02-20T21:43:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:43:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap6-clean 2026-02-20T21:43:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:43:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:43:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:43:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:43:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:43:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=11 2026-02-20T21:43:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:43:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:43:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 36815872 bytes 2026-02-20T21:43:48.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 72862720 bytes 2026-02-20T21:43:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 115792896 bytes 2026-02-20T21:43:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 196385792 bytes 2026-02-20T21:43:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 249562112 bytes 2026-02-20T21:43:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 275218432 bytes 2026-02-20T21:43:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 340748288 bytes 2026-02-20T21:43:49.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 364261376 bytes 2026-02-20T21:43:49.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 379898880 bytes 2026-02-20T21:43:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 450035712 bytes 2026-02-20T21:43:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 452531200 bytes 2026-02-20T21:43:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 481481728 bytes 2026-02-20T21:43:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 551107584 bytes 2026-02-20T21:43:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 664851456 bytes 2026-02-20T21:43:49.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 688521216 bytes 2026-02-20T21:43:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 698865664 bytes 2026-02-20T21:43:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 766611456 bytes 2026-02-20T21:43:49.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 803999744 bytes 2026-02-20T21:43:49.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 871557120 bytes 2026-02-20T21:43:49.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 877324288 bytes 2026-02-20T21:43:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 936948736 bytes 2026-02-20T21:43:49.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 964884480 bytes 2026-02-20T21:43:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 978188288 bytes 2026-02-20T21:43:49.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1024933888 bytes 2026-02-20T21:43:49.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1075685376 bytes 2026-02-20T21:43:49.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1103474688 bytes 2026-02-20T21:43:49.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1113193472 bytes 2026-02-20T21:43:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1140753408 bytes 2026-02-20T21:43:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1207600128 bytes 2026-02-20T21:43:50.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1226655744 bytes 2026-02-20T21:43:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1238667264 bytes 2026-02-20T21:43:50.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1290785792 bytes 2026-02-20T21:43:50.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1363501056 bytes 2026-02-20T21:43:50.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1402856448 bytes 2026-02-20T21:43:50.340 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1447693312 bytes 2026-02-20T21:43:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1493188608 bytes 2026-02-20T21:43:50.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1519433728 bytes 2026-02-20T21:43:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1578286080 bytes 2026-02-20T21:43:50.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1625707520 bytes 2026-02-20T21:43:50.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1651076096 bytes 2026-02-20T21:43:50.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1688386560 bytes 2026-02-20T21:43:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1708765184 bytes 2026-02-20T21:43:50.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1737639936 bytes 2026-02-20T21:43:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1778723840 bytes 2026-02-20T21:43:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1810553856 bytes 2026-02-20T21:43:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1866726400 bytes 2026-02-20T21:43:50.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1957052416 bytes 2026-02-20T21:43:50.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2015429632 bytes 2026-02-20T21:43:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2092598272 bytes 2026-02-20T21:43:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2122118144 bytes 2026-02-20T21:43:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2219990016 bytes 2026-02-20T21:43:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2284437504 bytes 2026-02-20T21:43:51.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2313403392 bytes 2026-02-20T21:43:51.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2348969984 bytes 2026-02-20T21:43:51.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2391435264 bytes 2026-02-20T21:43:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2409274368 bytes 2026-02-20T21:43:51.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2430196736 bytes 2026-02-20T21:43:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2475003904 bytes 2026-02-20T21:43:51.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2488605696 bytes 2026-02-20T21:43:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2489209856 bytes 2026-02-20T21:43:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2497961984 bytes 2026-02-20T21:43:51.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2513546240 bytes 2026-02-20T21:43:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2551961600 bytes 2026-02-20T21:43:51.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2572502016 bytes 2026-02-20T21:43:51.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2617027584 bytes 2026-02-20T21:43:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2635952128 bytes 2026-02-20T21:43:51.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2637817856 bytes 2026-02-20T21:43:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2653605888 bytes 2026-02-20T21:43:51.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2670841856 bytes 2026-02-20T21:43:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2726870016 bytes 2026-02-20T21:43:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2739909632 bytes 2026-02-20T21:43:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2752141312 bytes 2026-02-20T21:43:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2772145152 bytes 2026-02-20T21:43:51.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2799828992 bytes 2026-02-20T21:43:51.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2824856576 bytes 2026-02-20T21:43:51.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 2857583616 bytes 2026-02-20T21:43:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 2908070912 bytes 2026-02-20T21:43:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 2937922560 bytes 2026-02-20T21:43:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 2960322560 bytes 2026-02-20T21:43:52.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3019836416 bytes 2026-02-20T21:43:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3049387008 bytes 2026-02-20T21:43:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3092573184 bytes 2026-02-20T21:43:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3105169408 bytes 2026-02-20T21:43:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3138966528 bytes 2026-02-20T21:43:52.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3197699072 bytes 2026-02-20T21:43:52.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3245674496 bytes 2026-02-20T21:43:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3336218624 bytes 2026-02-20T21:43:52.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3389331456 bytes 2026-02-20T21:43:52.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3448336384 bytes 2026-02-20T21:43:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3501229056 bytes 2026-02-20T21:43:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3557892096 bytes 2026-02-20T21:43:52.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3558482944 bytes 2026-02-20T21:43:52.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3617951744 bytes 2026-02-20T21:43:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3683165184 bytes 2026-02-20T21:43:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3743706112 bytes 2026-02-20T21:43:52.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3744345088 bytes 2026-02-20T21:43:52.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3803336704 bytes 2026-02-20T21:43:52.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3857008640 bytes 2026-02-20T21:43:53.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3904055296 bytes 2026-02-20T21:43:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3932951552 bytes 2026-02-20T21:43:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 3984687104 bytes 2026-02-20T21:43:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4012429312 bytes 2026-02-20T21:43:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4046334976 bytes 2026-02-20T21:43:53.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4069998592 bytes 2026-02-20T21:43:53.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4125956096 bytes 2026-02-20T21:43:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4150575104 bytes 2026-02-20T21:43:53.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4221617152 bytes 2026-02-20T21:43:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4281392128 bytes 2026-02-20T21:43:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4303252480 bytes 2026-02-20T21:43:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4321586176 bytes 2026-02-20T21:43:53.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4343420928 bytes 2026-02-20T21:43:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4379405312 bytes 2026-02-20T21:43:53.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4463295488 bytes 2026-02-20T21:43:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4505428992 bytes 2026-02-20T21:43:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4541805568 bytes 2026-02-20T21:43:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4590166016 bytes 2026-02-20T21:43:53.799 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:43:53.791+0000 7fa5e2658640 -1 reset not still connected to 0x55b902d84750 2026-02-20T21:43:53.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4627785728 bytes 2026-02-20T21:43:53.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4659347456 bytes 2026-02-20T21:43:53.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4680375296 bytes 2026-02-20T21:43:53.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4714691584 bytes 2026-02-20T21:43:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4748725248 bytes 2026-02-20T21:43:54.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4800626688 bytes 2026-02-20T21:43:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4846331904 bytes 2026-02-20T21:43:54.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4846956544 bytes 2026-02-20T21:43:54.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4849086464 bytes 2026-02-20T21:43:54.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 4871299072 bytes 2026-02-20T21:43:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 4909755392 bytes 2026-02-20T21:43:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 4943518720 bytes 2026-02-20T21:43:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 4994677760 bytes 2026-02-20T21:43:54.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5068128256 bytes 2026-02-20T21:43:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5161906176 bytes 2026-02-20T21:43:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5223446528 bytes 2026-02-20T21:43:54.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5238671360 bytes 2026-02-20T21:43:54.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5248420864 bytes 2026-02-20T21:43:54.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5262541824 bytes 2026-02-20T21:43:54.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5295341568 bytes 2026-02-20T21:43:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5342911488 bytes 2026-02-20T21:43:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5379984384 bytes 2026-02-20T21:43:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5439724544 bytes 2026-02-20T21:43:54.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5529610240 bytes 2026-02-20T21:43:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5549294592 bytes 2026-02-20T21:43:54.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5574428672 bytes 2026-02-20T21:43:54.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5617983488 bytes 2026-02-20T21:43:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5727188992 bytes 2026-02-20T21:43:55.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5777373184 bytes 2026-02-20T21:43:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5800519680 bytes 2026-02-20T21:43:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5847224320 bytes 2026-02-20T21:43:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5881618432 bytes 2026-02-20T21:43:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5930552320 bytes 2026-02-20T21:43:55.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5961268224 bytes 2026-02-20T21:43:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6027857920 bytes 2026-02-20T21:43:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6080191488 bytes 2026-02-20T21:43:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6080776192 bytes 2026-02-20T21:43:55.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6138814464 bytes 2026-02-20T21:43:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6189237248 bytes 2026-02-20T21:43:55.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6215926784 bytes 2026-02-20T21:43:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6277726208 bytes 2026-02-20T21:43:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6358297600 bytes 2026-02-20T21:43:55.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6419401728 bytes 2026-02-20T21:43:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6458524672 bytes 2026-02-20T21:43:55.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6488853504 bytes 2026-02-20T21:43:55.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6508251136 bytes 2026-02-20T21:43:55.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6589540352 bytes 2026-02-20T21:43:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6595899392 bytes 2026-02-20T21:43:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6684388352 bytes 2026-02-20T21:43:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6734575616 bytes 2026-02-20T21:43:55.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6758973440 bytes 2026-02-20T21:43:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6776991744 bytes 2026-02-20T21:43:56.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6867962880 bytes 2026-02-20T21:43:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6874425344 bytes 2026-02-20T21:43:56.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6893181952 bytes 2026-02-20T21:43:56.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6909168640 bytes 2026-02-20T21:43:56.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 6994979840 bytes 2026-02-20T21:43:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 6995606528 bytes 2026-02-20T21:43:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7033428992 bytes 2026-02-20T21:43:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7087476736 bytes 2026-02-20T21:43:56.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7170729984 bytes 2026-02-20T21:43:56.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7265719296 bytes 2026-02-20T21:43:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7303678976 bytes 2026-02-20T21:43:56.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7401829376 bytes 2026-02-20T21:43:56.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7453437952 bytes 2026-02-20T21:43:56.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7503449088 bytes 2026-02-20T21:43:56.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7515197440 bytes 2026-02-20T21:43:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7523468288 bytes 2026-02-20T21:43:56.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7538091008 bytes 2026-02-20T21:43:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7569417216 bytes 2026-02-20T21:43:56.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7727089664 bytes 2026-02-20T21:43:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7831670784 bytes 2026-02-20T21:43:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7866670080 bytes 2026-02-20T21:43:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7947068416 bytes 2026-02-20T21:43:57.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8010790912 bytes 2026-02-20T21:43:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8033329152 bytes 2026-02-20T21:43:57.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8078100480 bytes 2026-02-20T21:43:57.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8100099072 bytes 2026-02-20T21:43:57.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8112791552 bytes 2026-02-20T21:43:57.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8123131904 bytes 2026-02-20T21:43:57.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8162959360 bytes 2026-02-20T21:43:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8173556736 bytes 2026-02-20T21:43:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8241285120 bytes 2026-02-20T21:43:57.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8241914880 bytes 2026-02-20T21:43:57.494 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9397196495520058) > weight (0.25) 2026-02-20T21:43:57.494 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8894848087912782) > weight (0.25) 2026-02-20T21:43:57.494 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7110666863319983) > weight (0.25) 2026-02-20T21:43:57.494 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.33032199201215184) > weight (0.25) 2026-02-20T21:43:57.494 INFO:tasks.rbd_mirror_thrash:waiting for 111.83136525442706 secs before thrashing 2026-02-20T21:43:57.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8272658432 bytes 2026-02-20T21:43:57.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8308058112 bytes 2026-02-20T21:43:57.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8360968192 bytes 2026-02-20T21:43:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8383791104 bytes 2026-02-20T21:43:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8395205632 bytes 2026-02-20T21:43:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8429060096 bytes 2026-02-20T21:43:57.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8431846400 bytes 2026-02-20T21:43:57.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8509995008 bytes 2026-02-20T21:43:57.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8528333824 bytes 2026-02-20T21:43:57.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8560324608 bytes 2026-02-20T21:43:57.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8576891904 bytes 2026-02-20T21:43:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8657233920 bytes 2026-02-20T21:43:58.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8765761536 bytes 2026-02-20T21:43:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8841006080 bytes 2026-02-20T21:43:58.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8887173120 bytes 2026-02-20T21:43:58.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 8930816000 bytes 2026-02-20T21:43:58.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 8950378496 bytes 2026-02-20T21:43:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 8972137472 bytes 2026-02-20T21:43:58.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 8979397632 bytes 2026-02-20T21:43:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9026439168 bytes 2026-02-20T21:43:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9066137600 bytes 2026-02-20T21:43:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9134840832 bytes 2026-02-20T21:43:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9173889024 bytes 2026-02-20T21:43:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9191455744 bytes 2026-02-20T21:43:58.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9208543232 bytes 2026-02-20T21:43:58.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9223216128 bytes 2026-02-20T21:43:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9300050944 bytes 2026-02-20T21:43:58.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9370180608 bytes 2026-02-20T21:43:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9447578624 bytes 2026-02-20T21:43:58.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9448201216 bytes 2026-02-20T21:43:58.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9461299200 bytes 2026-02-20T21:43:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9488602112 bytes 2026-02-20T21:43:58.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9519694848 bytes 2026-02-20T21:43:58.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9528250368 bytes 2026-02-20T21:43:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9543337984 bytes 2026-02-20T21:43:59.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 9609808896 bytes 2026-02-20T21:43:59.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 9610420224 bytes 2026-02-20T21:43:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 9641720832 bytes 2026-02-20T21:43:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 9694150656 bytes 2026-02-20T21:43:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 9770620928 bytes 2026-02-20T21:43:59.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 9771213824 bytes 2026-02-20T21:43:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 9811971072 bytes 2026-02-20T21:43:59.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 9889815552 bytes 2026-02-20T21:43:59.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 9944165376 bytes 2026-02-20T21:43:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10024004608 bytes 2026-02-20T21:43:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10092279808 bytes 2026-02-20T21:43:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10166430720 bytes 2026-02-20T21:43:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10205156352 bytes 2026-02-20T21:43:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:43:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:43:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:43:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:43:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:43:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:43:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:43:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:43:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:43:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:43:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:43:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:43:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:44:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:44:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:44:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:44:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:44:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:44:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:44:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:44:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:44:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:44:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:44:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":17884910.93,"bytes_per_snapshot":536547328.0,"last_snapshot_bytes":536547328,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623821,"remote_snapshot_timestamp":1771623821,"replay_state":"idle"} 2026-02-20T21:44:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.3 on trial107 2026-02-20T21:44:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:43:49 2026-02-20T21:44:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:44:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:44:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+stopped 2026-02-20T21:44:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: local image is primary 2026-02-20T21:44:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:43:31' 2026-02-20T21:44:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:44:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:44:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:44:00.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":17884910.93,"bytes_per_snapshot":536547328.0,"last_snapshot_bytes":536547328,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623821,"remote_snapshot_timestamp":1771623821,"replay_state":"idle"} 2026-02-20T21:44:00.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:44:00.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:44:00.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:44:00.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:44:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:44:00.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.3 on trial107 2026-02-20T21:44:00.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:44:00.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:44:00.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:43:49' 2026-02-20T21:44:00.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:43:49' 2026-02-20T21:44:00.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:00.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:00.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:00.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:44:00.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:43:49' 2026-02-20T21:44:00.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:44:00.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:44:00.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:44:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='OKOKOK142154266mirror.1trial10720.2.0-677-gf7870454falseOK42234272mirror.3trial10720.2.0-677-gf7870454trueOK43894391mirror.2trial10720.2.0-677-gf7870454falseOK44454448mirror.0trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":17884910.93,"bytes_per_snapshot":536547328.0,"last_snapshot_bytes":536547328,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623821,"remote_snapshot_timestamp":1771623821,"replay_state":"idle"}42234272mirror.3trial1072026-02-20 21:43:49cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911dup+stoppedlocal image is primary2026-02-20 21:43:31' 2026-02-20T21:44:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:44:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:44:00.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:43:49' 2026-02-20T21:44:00.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:44:00.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:44:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:44:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:44:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:44:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap7 2026-02-20T21:44:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap7 2026-02-20T21:44:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:44:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap7 2026-02-20T21:44:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:44:27.230 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4989782079640127) > weight (0.25) 2026-02-20T21:44:27.230 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3092595394297404) > weight (0.25) 2026-02-20T21:44:27.230 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9418612351178546) > weight (0.25) 2026-02-20T21:44:27.230 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.4480238384149988) > weight (0.25) 2026-02-20T21:44:27.230 INFO:tasks.rbd_mirror_thrash:waiting for 114.88050850494122 secs before thrashing 2026-02-20T21:44:30.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:44:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:44:30.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:44:31.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:44:31.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:44:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:44:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:44:31.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:44:31.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:44:31.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:44:31.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:44:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:44:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:44:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:44:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:44:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:44:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:44:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:44:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:44:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:44:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:44:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:44:31.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:44:31.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:44:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:44:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:44:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:44:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:44:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:44:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:44:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:44:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:44:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:44:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:44:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:44:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:44:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:44:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:44:31.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:44:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:44:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:44:31.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:44:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:44:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:44:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:44:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:44:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:44:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:44:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:44:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:44:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:44:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:44:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:44:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:44:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:44:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:44:31.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:44:31.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:44:31.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:44:31.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:44:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:44:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:44:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:44:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:44:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:44:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:44:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:44:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:44:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:44:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:44:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:44:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:44:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:44:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:44:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:44:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:44:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:44:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:44:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:44:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:44:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:44:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 23 2026-02-20T21:44:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:44:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:44:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:44:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:44:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:44:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:44:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:44:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:44:33.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:44:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:44:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:44:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:44:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:44:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:44:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:44:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:44:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:44:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:44:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:44:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=23 2026-02-20T21:44:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:44:33.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:44:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:44:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:44:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:44:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:44:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:44:33.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:44:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:44:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:44:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:44:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:44:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 23 = '' 2026-02-20T21:44:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:44:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:44:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:44:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:44:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:44:34.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:44:34.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:44:34.359 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:44:34.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:44:34.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:44:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:44:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:44:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 23 = '' 2026-02-20T21:44:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:44:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:44:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:44:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:44:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:44:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:44:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:44:35.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:44:35.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:44:35.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:44:35.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:44:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:44:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 23 = '' 2026-02-20T21:44:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:44:35.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:44:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:44:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:44:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:44:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:44:36.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:44:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=23 2026-02-20T21:44:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 23 = 23 2026-02-20T21:44:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2026-02-20T21:44:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap7 2026-02-20T21:44:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:44:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:44:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:44:37.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2026-02-20T21:44:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:44:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:44:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:44:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 14 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:43:45 2026 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:44:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:44:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:44:37.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:44:37.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snap6-clean ']' 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap6-clean 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:44:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:44:38.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:44:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap7 2026-02-20T21:44:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap7 2026-02-20T21:44:39.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:44:39.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:44:39.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:39.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:39.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:44:39.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:44:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:40.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:44:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:40.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:44:40.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap7-clean 2026-02-20T21:44:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap7-clean 2026-02-20T21:44:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:44:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap7-clean 2026-02-20T21:44:40.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:44:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:44:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:44:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:44:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:44:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=13 2026-02-20T21:44:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:44:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:44:41.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 40566784 bytes 2026-02-20T21:44:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 102079488 bytes 2026-02-20T21:44:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 168723456 bytes 2026-02-20T21:44:42.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 193304576 bytes 2026-02-20T21:44:42.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 267799552 bytes 2026-02-20T21:44:42.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 302359552 bytes 2026-02-20T21:44:42.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 318215168 bytes 2026-02-20T21:44:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 357481472 bytes 2026-02-20T21:44:42.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 381263872 bytes 2026-02-20T21:44:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 461755392 bytes 2026-02-20T21:44:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 486946816 bytes 2026-02-20T21:44:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 501161984 bytes 2026-02-20T21:44:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 553043968 bytes 2026-02-20T21:44:42.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 599437312 bytes 2026-02-20T21:44:42.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 624800768 bytes 2026-02-20T21:44:42.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 641126400 bytes 2026-02-20T21:44:42.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 664377344 bytes 2026-02-20T21:44:42.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 670977024 bytes 2026-02-20T21:44:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 716900352 bytes 2026-02-20T21:44:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 768692224 bytes 2026-02-20T21:44:42.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 786851840 bytes 2026-02-20T21:44:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 788573184 bytes 2026-02-20T21:44:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 815444992 bytes 2026-02-20T21:44:42.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 823191552 bytes 2026-02-20T21:44:42.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 860375040 bytes 2026-02-20T21:44:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 913049600 bytes 2026-02-20T21:44:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 913673216 bytes 2026-02-20T21:44:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 955520000 bytes 2026-02-20T21:44:43.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 991555584 bytes 2026-02-20T21:44:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1053517824 bytes 2026-02-20T21:44:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1090308096 bytes 2026-02-20T21:44:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1129184256 bytes 2026-02-20T21:44:43.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1224914944 bytes 2026-02-20T21:44:43.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1246013440 bytes 2026-02-20T21:44:43.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1266245632 bytes 2026-02-20T21:44:43.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1318062080 bytes 2026-02-20T21:44:43.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1377538048 bytes 2026-02-20T21:44:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1413874688 bytes 2026-02-20T21:44:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1457904640 bytes 2026-02-20T21:44:43.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1526942720 bytes 2026-02-20T21:44:43.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1527572480 bytes 2026-02-20T21:44:43.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1571700736 bytes 2026-02-20T21:44:43.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1659081728 bytes 2026-02-20T21:44:43.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1666964480 bytes 2026-02-20T21:44:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1740144640 bytes 2026-02-20T21:44:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1771965440 bytes 2026-02-20T21:44:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1787443200 bytes 2026-02-20T21:44:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1846219776 bytes 2026-02-20T21:44:43.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1872340992 bytes 2026-02-20T21:44:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1936222208 bytes 2026-02-20T21:44:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2006504448 bytes 2026-02-20T21:44:44.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2034104320 bytes 2026-02-20T21:44:44.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2034705408 bytes 2026-02-20T21:44:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2053702656 bytes 2026-02-20T21:44:44.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2096711680 bytes 2026-02-20T21:44:44.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2174605312 bytes 2026-02-20T21:44:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2259799040 bytes 2026-02-20T21:44:44.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2292071424 bytes 2026-02-20T21:44:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2329056256 bytes 2026-02-20T21:44:44.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2338561024 bytes 2026-02-20T21:44:44.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2361009152 bytes 2026-02-20T21:44:44.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2382718976 bytes 2026-02-20T21:44:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2449400832 bytes 2026-02-20T21:44:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2472782848 bytes 2026-02-20T21:44:44.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2500201472 bytes 2026-02-20T21:44:44.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2519864320 bytes 2026-02-20T21:44:44.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2529326080 bytes 2026-02-20T21:44:44.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2577543168 bytes 2026-02-20T21:44:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2639446016 bytes 2026-02-20T21:44:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2684120064 bytes 2026-02-20T21:44:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2717637632 bytes 2026-02-20T21:44:44.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2740220928 bytes 2026-02-20T21:44:45.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2816175104 bytes 2026-02-20T21:44:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2859215872 bytes 2026-02-20T21:44:45.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2913508352 bytes 2026-02-20T21:44:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3006592000 bytes 2026-02-20T21:44:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3049684992 bytes 2026-02-20T21:44:45.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3050943488 bytes 2026-02-20T21:44:45.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3085875200 bytes 2026-02-20T21:44:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3128211456 bytes 2026-02-20T21:44:45.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3145805824 bytes 2026-02-20T21:44:45.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3234088960 bytes 2026-02-20T21:44:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3280143360 bytes 2026-02-20T21:44:45.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3343083520 bytes 2026-02-20T21:44:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3347224576 bytes 2026-02-20T21:44:45.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3349293056 bytes 2026-02-20T21:44:45.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3349905408 bytes 2026-02-20T21:44:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3387821056 bytes 2026-02-20T21:44:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3424034816 bytes 2026-02-20T21:44:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3465838592 bytes 2026-02-20T21:44:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3490286592 bytes 2026-02-20T21:44:45.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3518580736 bytes 2026-02-20T21:44:45.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3594593280 bytes 2026-02-20T21:44:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3640877056 bytes 2026-02-20T21:44:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3684208640 bytes 2026-02-20T21:44:46.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3715466240 bytes 2026-02-20T21:44:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3758098432 bytes 2026-02-20T21:44:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3777623040 bytes 2026-02-20T21:44:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3820051456 bytes 2026-02-20T21:44:46.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3875858432 bytes 2026-02-20T21:44:46.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 3892702208 bytes 2026-02-20T21:44:46.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 3916156928 bytes 2026-02-20T21:44:46.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 3992698880 bytes 2026-02-20T21:44:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4025115648 bytes 2026-02-20T21:44:46.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4091919360 bytes 2026-02-20T21:44:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4110709760 bytes 2026-02-20T21:44:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4132315136 bytes 2026-02-20T21:44:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4215134208 bytes 2026-02-20T21:44:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4242722816 bytes 2026-02-20T21:44:46.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4280823808 bytes 2026-02-20T21:44:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4326387712 bytes 2026-02-20T21:44:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4371120128 bytes 2026-02-20T21:44:46.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4405703680 bytes 2026-02-20T21:44:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4406304768 bytes 2026-02-20T21:44:47.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4451859456 bytes 2026-02-20T21:44:47.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4493681664 bytes 2026-02-20T21:44:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4530204672 bytes 2026-02-20T21:44:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4613764096 bytes 2026-02-20T21:44:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4630918144 bytes 2026-02-20T21:44:47.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4728637440 bytes 2026-02-20T21:44:47.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4785158144 bytes 2026-02-20T21:44:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4811583488 bytes 2026-02-20T21:44:47.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4827492352 bytes 2026-02-20T21:44:47.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4956134400 bytes 2026-02-20T21:44:47.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4964869120 bytes 2026-02-20T21:44:47.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 4996077568 bytes 2026-02-20T21:44:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5027454976 bytes 2026-02-20T21:44:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5100806144 bytes 2026-02-20T21:44:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5133290496 bytes 2026-02-20T21:44:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5159371776 bytes 2026-02-20T21:44:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5228416000 bytes 2026-02-20T21:44:47.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5273461760 bytes 2026-02-20T21:44:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5326737408 bytes 2026-02-20T21:44:47.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5395728384 bytes 2026-02-20T21:44:47.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5396348928 bytes 2026-02-20T21:44:48.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5466557440 bytes 2026-02-20T21:44:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5504971776 bytes 2026-02-20T21:44:48.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5591644160 bytes 2026-02-20T21:44:48.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5618700288 bytes 2026-02-20T21:44:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5708070912 bytes 2026-02-20T21:44:48.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5717004288 bytes 2026-02-20T21:44:48.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5776192512 bytes 2026-02-20T21:44:48.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5821407232 bytes 2026-02-20T21:44:48.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5876983808 bytes 2026-02-20T21:44:48.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5945106432 bytes 2026-02-20T21:44:48.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5991860224 bytes 2026-02-20T21:44:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6062553088 bytes 2026-02-20T21:44:48.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6084003840 bytes 2026-02-20T21:44:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6123298816 bytes 2026-02-20T21:44:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6129530880 bytes 2026-02-20T21:44:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6181556224 bytes 2026-02-20T21:44:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6213911552 bytes 2026-02-20T21:44:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6231708672 bytes 2026-02-20T21:44:48.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6252776448 bytes 2026-02-20T21:44:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6290225152 bytes 2026-02-20T21:44:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6329591808 bytes 2026-02-20T21:44:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6390731776 bytes 2026-02-20T21:44:49.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6446414848 bytes 2026-02-20T21:44:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6471021568 bytes 2026-02-20T21:44:49.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6504809472 bytes 2026-02-20T21:44:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6534871040 bytes 2026-02-20T21:44:49.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6600159232 bytes 2026-02-20T21:44:49.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6605457408 bytes 2026-02-20T21:44:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6636919808 bytes 2026-02-20T21:44:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6655926272 bytes 2026-02-20T21:44:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6697229312 bytes 2026-02-20T21:44:49.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6742632448 bytes 2026-02-20T21:44:49.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6783895552 bytes 2026-02-20T21:44:49.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6835651584 bytes 2026-02-20T21:44:49.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6911608832 bytes 2026-02-20T21:44:49.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6956944384 bytes 2026-02-20T21:44:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6990114816 bytes 2026-02-20T21:44:49.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7037809664 bytes 2026-02-20T21:44:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7062981632 bytes 2026-02-20T21:44:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7108892672 bytes 2026-02-20T21:44:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7164135424 bytes 2026-02-20T21:44:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7276563456 bytes 2026-02-20T21:44:50.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7303317504 bytes 2026-02-20T21:44:50.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7363170304 bytes 2026-02-20T21:44:50.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7408217088 bytes 2026-02-20T21:44:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7427736576 bytes 2026-02-20T21:44:50.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7483536384 bytes 2026-02-20T21:44:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7521524736 bytes 2026-02-20T21:44:50.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7543249920 bytes 2026-02-20T21:44:50.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7593634816 bytes 2026-02-20T21:44:50.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7635716096 bytes 2026-02-20T21:44:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7676017664 bytes 2026-02-20T21:44:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7743263744 bytes 2026-02-20T21:44:50.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7794844672 bytes 2026-02-20T21:44:50.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7927545856 bytes 2026-02-20T21:44:50.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7965001728 bytes 2026-02-20T21:44:50.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7990386688 bytes 2026-02-20T21:44:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8038937600 bytes 2026-02-20T21:44:51.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8065334272 bytes 2026-02-20T21:44:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8106067968 bytes 2026-02-20T21:44:51.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8116568064 bytes 2026-02-20T21:44:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8167478272 bytes 2026-02-20T21:44:51.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8238518272 bytes 2026-02-20T21:44:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8279077888 bytes 2026-02-20T21:44:51.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8322933760 bytes 2026-02-20T21:44:51.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8370021376 bytes 2026-02-20T21:44:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8473451520 bytes 2026-02-20T21:44:51.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8537637888 bytes 2026-02-20T21:44:51.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8597624832 bytes 2026-02-20T21:44:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8641062912 bytes 2026-02-20T21:44:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8658899968 bytes 2026-02-20T21:44:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8701049856 bytes 2026-02-20T21:44:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8708169728 bytes 2026-02-20T21:44:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8774679552 bytes 2026-02-20T21:44:51.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8805022720 bytes 2026-02-20T21:44:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8826068992 bytes 2026-02-20T21:44:51.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8904217600 bytes 2026-02-20T21:44:52.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8943411200 bytes 2026-02-20T21:44:52.075 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8978768896 bytes 2026-02-20T21:44:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9026651136 bytes 2026-02-20T21:44:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9060061184 bytes 2026-02-20T21:44:52.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9103608832 bytes 2026-02-20T21:44:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9174620160 bytes 2026-02-20T21:44:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9209507840 bytes 2026-02-20T21:44:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9252408320 bytes 2026-02-20T21:44:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9344922624 bytes 2026-02-20T21:44:52.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9395432448 bytes 2026-02-20T21:44:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9418558464 bytes 2026-02-20T21:44:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9486329856 bytes 2026-02-20T21:44:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9516209152 bytes 2026-02-20T21:44:52.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9580147712 bytes 2026-02-20T21:44:52.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9594447872 bytes 2026-02-20T21:44:52.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9673831424 bytes 2026-02-20T21:44:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9762087936 bytes 2026-02-20T21:44:52.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9834341376 bytes 2026-02-20T21:44:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9878365184 bytes 2026-02-20T21:44:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9923218432 bytes 2026-02-20T21:44:53.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9958026240 bytes 2026-02-20T21:44:53.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9981216768 bytes 2026-02-20T21:44:53.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9995928576 bytes 2026-02-20T21:44:53.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 10007427072 bytes 2026-02-20T21:44:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 10041291776 bytes 2026-02-20T21:44:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 10056957952 bytes 2026-02-20T21:44:53.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 10079020032 bytes 2026-02-20T21:44:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 10144380928 bytes 2026-02-20T21:44:53.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 10159315968 bytes 2026-02-20T21:44:53.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10201426944 bytes 2026-02-20T21:44:53.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 10237904896 bytes 2026-02-20T21:44:53.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 10339414016 bytes 2026-02-20T21:44:53.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10381278208 bytes 2026-02-20T21:44:53.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10427884544 bytes 2026-02-20T21:44:53.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10471031808 bytes 2026-02-20T21:44:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10483818496 bytes 2026-02-20T21:44:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 10541659136 bytes 2026-02-20T21:44:53.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 10569193472 bytes 2026-02-20T21:44:53.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25100 IOs, 10586531840 bytes 2026-02-20T21:44:53.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25200 IOs, 10642324480 bytes 2026-02-20T21:44:53.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25300 IOs, 10671131648 bytes 2026-02-20T21:44:53.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25400 IOs, 10711341056 bytes 2026-02-20T21:44:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25500 IOs, 10745089024 bytes 2026-02-20T21:44:54.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25600 IOs, 10776461312 bytes 2026-02-20T21:44:54.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25700 IOs, 10862469120 bytes 2026-02-20T21:44:54.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25800 IOs, 10926581760 bytes 2026-02-20T21:44:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25900 IOs, 11016907776 bytes 2026-02-20T21:44:54.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26000 IOs, 11058950144 bytes 2026-02-20T21:44:54.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26100 IOs, 11059561472 bytes 2026-02-20T21:44:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26200 IOs, 11104622592 bytes 2026-02-20T21:44:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26300 IOs, 11183086592 bytes 2026-02-20T21:44:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26400 IOs, 11196135424 bytes 2026-02-20T21:44:54.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26500 IOs, 11197308928 bytes 2026-02-20T21:44:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26600 IOs, 11267881984 bytes 2026-02-20T21:44:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26700 IOs, 11274935296 bytes 2026-02-20T21:44:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:44:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:44:54.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:44:54.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:44:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:44:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:44:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:44:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:44:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:44:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:44:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:44:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:44:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:44:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:44:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":17885047.47,"bytes_per_snapshot":536549376.0,"last_snapshot_bytes":536551424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623872,"remote_snapshot_timestamp":1771623872,"replay_state":"idle"} 2026-02-20T21:44:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.3 on trial107 2026-02-20T21:44:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:44:49 2026-02-20T21:44:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:44:55.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:44:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+stopped 2026-02-20T21:44:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: local image is primary 2026-02-20T21:44:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:44:31' 2026-02-20T21:44:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:44:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:44:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:44:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":17885047.47,"bytes_per_snapshot":536549376.0,"last_snapshot_bytes":536551424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623872,"remote_snapshot_timestamp":1771623872,"replay_state":"idle"} 2026-02-20T21:44:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:44:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:44:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:44:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:44:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:44:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.3 on trial107 2026-02-20T21:44:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:44:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:44:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:44:49' 2026-02-20T21:44:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:44:49' 2026-02-20T21:44:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:44:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:55.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:44:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:44:49' 2026-02-20T21:44:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:44:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:44:55.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:44:55.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='OKOKOK142154266mirror.1trial10720.2.0-677-gf7870454falseOK42234272mirror.3trial10720.2.0-677-gf7870454trueOK43894391mirror.2trial10720.2.0-677-gf7870454falseOK44454448mirror.0trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":17885047.47,"bytes_per_snapshot":536549376.0,"last_snapshot_bytes":536551424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623872,"remote_snapshot_timestamp":1771623872,"replay_state":"idle"}42234272mirror.3trial1072026-02-20 21:44:49cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911dup+stoppedlocal image is primary2026-02-20 21:44:31' 2026-02-20T21:44:55.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:44:55.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:44:55.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:44:49' 2026-02-20T21:44:55.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:44:55.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:44:55.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:44:55.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:44:55.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:44:55.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:44:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap8 2026-02-20T21:44:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap8 2026-02-20T21:44:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:44:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:44:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:44:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap8 2026-02-20T21:44:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:45:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:45:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:45:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:45:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:45:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:45:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:45:26.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:45:26.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:45:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:45:26.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:45:26.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:45:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:45:26.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:45:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:45:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:45:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:45:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:45:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:45:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:45:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:45:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:45:26.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:45:26.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:45:26.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:45:26.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:45:26.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:45:26.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:45:26.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:45:26.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:45:26.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:45:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:45:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:45:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:45:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:45:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:45:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:45:26.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:45:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:45:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:45:26.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:45:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:45:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:45:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:45:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:45:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:45:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:45:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:45:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:45:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:45:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:45:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:45:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:45:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:45:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:45:26.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:45:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:45:26.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:45:26.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:45:26.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:45:26.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:45:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:45:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:45:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:45:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:45:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:45:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:45:26.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:45:26.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:45:26.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:45:26.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:45:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:45:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:45:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:45:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:45:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:45:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:45:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:45:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:45:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:45:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:45:26.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:45:26.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:45:26.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:45:26.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:45:26.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:45:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:45:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:45:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:45:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:45:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:45:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 26 2026-02-20T21:45:28.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:45:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:45:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:45:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:45:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:45:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:45:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:45:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:45:28.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:45:28.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:45:28.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:45:28.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:45:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:45:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:45:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:45:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:45:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:45:28.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:45:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:45:28.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=26 2026-02-20T21:45:28.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:45:28.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:45:28.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:45:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:45:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:45:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:45:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:45:28.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:45:28.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:45:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:45:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:45:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:45:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:45:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 26 = '' 2026-02-20T21:45:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:45:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:45:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:45:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:45:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:45:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:45:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:45:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:45:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:45:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:45:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:45:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:45:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 26 = '' 2026-02-20T21:45:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:45:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:45:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:45:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:45:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:45:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:45:30.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:45:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=26 2026-02-20T21:45:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 26 = 26 2026-02-20T21:45:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:45:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2026-02-20T21:45:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap8 2026-02-20T21:45:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:45:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:45:30.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:45:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2026-02-20T21:45:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:45:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:45:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:45:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 16 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:43:45 2026 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:45:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:45:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:45:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snap7-clean ']' 2026-02-20T21:45:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-20T21:45:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap7-clean 2026-02-20T21:45:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:45:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:45:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:31.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:45:31.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:45:31.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:32.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:45:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap8 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap8 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:45:33.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:45:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:45:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:33.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:45:34.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:45:34.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap8-clean 2026-02-20T21:45:34.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap8-clean 2026-02-20T21:45:34.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:45:34.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:34.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:34.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap8-clean 2026-02-20T21:45:34.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:45:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:45:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:45:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:45:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:45:34.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:45:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:45:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:45:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 19301376 bytes 2026-02-20T21:45:34.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 107829248 bytes 2026-02-20T21:45:35.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 122018816 bytes 2026-02-20T21:45:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 128926720 bytes 2026-02-20T21:45:35.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 147376128 bytes 2026-02-20T21:45:35.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 187824128 bytes 2026-02-20T21:45:35.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 202443776 bytes 2026-02-20T21:45:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 240731136 bytes 2026-02-20T21:45:35.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 285814784 bytes 2026-02-20T21:45:35.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 334290944 bytes 2026-02-20T21:45:35.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 365322240 bytes 2026-02-20T21:45:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 461180928 bytes 2026-02-20T21:45:35.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 520097792 bytes 2026-02-20T21:45:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 545372160 bytes 2026-02-20T21:45:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 566737920 bytes 2026-02-20T21:45:35.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 629377024 bytes 2026-02-20T21:45:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 730217472 bytes 2026-02-20T21:45:35.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 743631872 bytes 2026-02-20T21:45:35.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 792936448 bytes 2026-02-20T21:45:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 839309312 bytes 2026-02-20T21:45:35.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 887200768 bytes 2026-02-20T21:45:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 951845888 bytes 2026-02-20T21:45:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 984285184 bytes 2026-02-20T21:45:36.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1023304704 bytes 2026-02-20T21:45:36.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1056680960 bytes 2026-02-20T21:45:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1098853376 bytes 2026-02-20T21:45:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1139678208 bytes 2026-02-20T21:45:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1209402368 bytes 2026-02-20T21:45:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1252046848 bytes 2026-02-20T21:45:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1262633984 bytes 2026-02-20T21:45:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1295428608 bytes 2026-02-20T21:45:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1307531264 bytes 2026-02-20T21:45:36.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1333685248 bytes 2026-02-20T21:45:36.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1350671360 bytes 2026-02-20T21:45:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1371249664 bytes 2026-02-20T21:45:36.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1400754176 bytes 2026-02-20T21:45:36.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1442881536 bytes 2026-02-20T21:45:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1523398656 bytes 2026-02-20T21:45:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1553902592 bytes 2026-02-20T21:45:36.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1588440064 bytes 2026-02-20T21:45:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1610031104 bytes 2026-02-20T21:45:36.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1655287808 bytes 2026-02-20T21:45:36.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1718920192 bytes 2026-02-20T21:45:36.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1739146240 bytes 2026-02-20T21:45:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1821334528 bytes 2026-02-20T21:45:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1857033216 bytes 2026-02-20T21:45:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1909073920 bytes 2026-02-20T21:45:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1946466304 bytes 2026-02-20T21:45:37.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1962468352 bytes 2026-02-20T21:45:37.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1994062848 bytes 2026-02-20T21:45:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2013054976 bytes 2026-02-20T21:45:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2023855104 bytes 2026-02-20T21:45:37.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2085444608 bytes 2026-02-20T21:45:37.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2145089536 bytes 2026-02-20T21:45:37.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2199659520 bytes 2026-02-20T21:45:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2215931904 bytes 2026-02-20T21:45:37.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2267157504 bytes 2026-02-20T21:45:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2346823680 bytes 2026-02-20T21:45:37.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2373095424 bytes 2026-02-20T21:45:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2403315712 bytes 2026-02-20T21:45:37.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2466839552 bytes 2026-02-20T21:45:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2480969728 bytes 2026-02-20T21:45:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2481571840 bytes 2026-02-20T21:45:37.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2527699968 bytes 2026-02-20T21:45:37.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2587969536 bytes 2026-02-20T21:45:38.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2646530048 bytes 2026-02-20T21:45:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2670360576 bytes 2026-02-20T21:45:38.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2714929152 bytes 2026-02-20T21:45:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2740120576 bytes 2026-02-20T21:45:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2767215616 bytes 2026-02-20T21:45:38.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2802298880 bytes 2026-02-20T21:45:38.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2849974272 bytes 2026-02-20T21:45:38.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2885908480 bytes 2026-02-20T21:45:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2911570944 bytes 2026-02-20T21:45:38.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2941745152 bytes 2026-02-20T21:45:38.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 2944707584 bytes 2026-02-20T21:45:38.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3022812160 bytes 2026-02-20T21:45:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3065464832 bytes 2026-02-20T21:45:38.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3094359040 bytes 2026-02-20T21:45:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3210485760 bytes 2026-02-20T21:45:38.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3257573376 bytes 2026-02-20T21:45:38.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3287115776 bytes 2026-02-20T21:45:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3287741440 bytes 2026-02-20T21:45:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3327805440 bytes 2026-02-20T21:45:38.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3364978688 bytes 2026-02-20T21:45:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3410546688 bytes 2026-02-20T21:45:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3504473088 bytes 2026-02-20T21:45:39.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3530970112 bytes 2026-02-20T21:45:39.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3558825984 bytes 2026-02-20T21:45:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3587732480 bytes 2026-02-20T21:45:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3642907648 bytes 2026-02-20T21:45:39.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3672855552 bytes 2026-02-20T21:45:39.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3690077184 bytes 2026-02-20T21:45:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3702228992 bytes 2026-02-20T21:45:39.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3715081216 bytes 2026-02-20T21:45:39.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3771697152 bytes 2026-02-20T21:45:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3772329984 bytes 2026-02-20T21:45:39.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3812406272 bytes 2026-02-20T21:45:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3830436864 bytes 2026-02-20T21:45:39.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3859793920 bytes 2026-02-20T21:45:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 3891284992 bytes 2026-02-20T21:45:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 3907479552 bytes 2026-02-20T21:45:39.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 3938091008 bytes 2026-02-20T21:45:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 3964487680 bytes 2026-02-20T21:45:39.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4013428736 bytes 2026-02-20T21:45:39.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4023848960 bytes 2026-02-20T21:45:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4079838208 bytes 2026-02-20T21:45:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4116528128 bytes 2026-02-20T21:45:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4127869952 bytes 2026-02-20T21:45:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4190334976 bytes 2026-02-20T21:45:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4215729152 bytes 2026-02-20T21:45:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4255478784 bytes 2026-02-20T21:45:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4266490880 bytes 2026-02-20T21:45:40.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4303331328 bytes 2026-02-20T21:45:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4322330624 bytes 2026-02-20T21:45:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4367628288 bytes 2026-02-20T21:45:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4465746944 bytes 2026-02-20T21:45:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4466371584 bytes 2026-02-20T21:45:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4537136128 bytes 2026-02-20T21:45:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4548969472 bytes 2026-02-20T21:45:40.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4552028160 bytes 2026-02-20T21:45:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4641796096 bytes 2026-02-20T21:45:40.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4679265280 bytes 2026-02-20T21:45:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4679886848 bytes 2026-02-20T21:45:40.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4785731584 bytes 2026-02-20T21:45:40.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:45:40.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:45:40.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:45:40.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:45:40.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:45:40.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:40.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:40.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:40.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:45:40.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:45:40.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:45:40.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:45:40.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:45:41.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:45:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:45:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:45:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:45:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:45:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:45:41.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:45:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:45:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:45:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:45:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536549376.0,"last_snapshot_bytes":536551424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623872,"remote_snapshot_timestamp":1771623872,"replay_state":"idle"} 2026-02-20T21:45:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.3 on trial107 2026-02-20T21:45:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:45:19 2026-02-20T21:45:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:45:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:45:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+stopped 2026-02-20T21:45:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: local image is primary 2026-02-20T21:45:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:45:31' 2026-02-20T21:45:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:45:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:45:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:45:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536549376.0,"last_snapshot_bytes":536551424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623872,"remote_snapshot_timestamp":1771623872,"replay_state":"idle"} 2026-02-20T21:45:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:45:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:45:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:45:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:45:41.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:45:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.3 on trial107 2026-02-20T21:45:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:45:41.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:45:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:45:19' 2026-02-20T21:45:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:45:19' 2026-02-20T21:45:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:45:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:45:41.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:45:19' 2026-02-20T21:45:41.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:45:41.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:45:41.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:45:41.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='OKOKOK142154266mirror.1trial10720.2.0-677-gf7870454falseOK42234272mirror.3trial10720.2.0-677-gf7870454trueOK43894391mirror.2trial10720.2.0-677-gf7870454falseOK44454448mirror.0trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":536549376.0,"last_snapshot_bytes":536551424,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623872,"remote_snapshot_timestamp":1771623872,"replay_state":"idle"}42234272mirror.3trial1072026-02-20 21:45:19cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911dup+stoppedlocal image is primary2026-02-20 21:45:31' 2026-02-20T21:45:41.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:45:41.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:45:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:45:19' 2026-02-20T21:45:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:45:41.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:45:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:45:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:45:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:45:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:45:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:45:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap9 2026-02-20T21:45:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap9 2026-02-20T21:45:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:45:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:45:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:45:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap9 2026-02-20T21:45:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:45:49.327 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:45:49.327 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:45:49.328 INFO:tasks.rbd_mirror_thrash:waiting for 4.075546572129421 secs before reviving daemons 2026-02-20T21:45:49.328 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T21:45:49.325+0000 7f135310b640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 23295) UID: 1000 2026-02-20T21:45:53.404 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:45:53.404 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:45:53.405 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:45:53.405 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:45:53.405 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:45:53.406 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:45:53.406 INFO:tasks.rbd_mirror_thrash:waiting for 72.68225940612155 secs before thrashing 2026-02-20T21:46:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:46:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:46:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:46:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:46:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:46:11.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:46:12.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:46:12.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:12.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:12.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:46:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:46:12.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:46:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:46:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:46:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:46:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:46:12.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:46:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:46:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:46:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:46:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:46:12.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:46:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:46:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:46:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:46:12.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:46:12.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:46:12.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:46:12.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:46:12.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:46:12.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:46:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:46:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:46:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:46:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:46:12.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:46:12.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:46:12.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:46:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:46:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:46:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:46:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:46:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:46:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:46:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:46:12.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:46:12.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:46:12.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:46:12.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:46:12.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:46:12.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:12.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:46:12.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:46:12.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:12.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:46:12.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:46:12.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:46:12.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:46:12.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:46:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:46:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:46:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:46:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:46:12.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:46:12.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:46:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:46:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:46:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:46:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:46:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:46:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:46:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:46:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:46:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:46:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:46:12.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:46:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:46:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:46:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:46:12.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:46:12.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:46:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:46:12.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:12.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:46:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 29 2026-02-20T21:46:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:46:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:46:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:46:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:46:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:46:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:46:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:46:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:46:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:46:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:46:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:46:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:46:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:46:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:46:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:46:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:46:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:46:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:46:14.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:46:14.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=29 2026-02-20T21:46:14.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:46:14.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:46:14.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:46:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:46:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:46:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:46:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:46:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:46:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:46:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:46:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:46:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:46:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:46:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 29 = '' 2026-02-20T21:46:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:46:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:46:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:46:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:46:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:46:15.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:46:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:46:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:46:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:46:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:46:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:46:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:46:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 29 = '' 2026-02-20T21:46:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:46:15.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:46:16.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:46:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:46:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:46:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:46:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:46:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=29 2026-02-20T21:46:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 29 = 29 2026-02-20T21:46:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:46:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2026-02-20T21:46:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap9 2026-02-20T21:46:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:46:16.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:46:16.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:46:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2026-02-20T21:46:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:46:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:46:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:46:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 18 2026-02-20T21:46:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:46:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:46:17.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:45:31 2026 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:46:17.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:46:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:46:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snap8-clean ']' 2026-02-20T21:46:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-20T21:46:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:17.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap8-clean 2026-02-20T21:46:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:46:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:46:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:17.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:46:17.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:46:17.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:18.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:46:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:46:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap9 2026-02-20T21:46:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap9 2026-02-20T21:46:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:46:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:46:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:46:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:46:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:19.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:46:19.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:20.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:46:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:46:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap9-clean 2026-02-20T21:46:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap9-clean 2026-02-20T21:46:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:46:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap9-clean 2026-02-20T21:46:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:46:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:46:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:46:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:46:20.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 56212480 bytes 2026-02-20T21:46:20.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 67601408 bytes 2026-02-20T21:46:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 77117440 bytes 2026-02-20T21:46:20.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 89629696 bytes 2026-02-20T21:46:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 135565312 bytes 2026-02-20T21:46:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 176327680 bytes 2026-02-20T21:46:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 216261632 bytes 2026-02-20T21:46:21.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 225300480 bytes 2026-02-20T21:46:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 250862592 bytes 2026-02-20T21:46:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 350721024 bytes 2026-02-20T21:46:21.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 394288128 bytes 2026-02-20T21:46:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 404825088 bytes 2026-02-20T21:46:21.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 432596992 bytes 2026-02-20T21:46:21.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 475755520 bytes 2026-02-20T21:46:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 490912768 bytes 2026-02-20T21:46:21.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 510229504 bytes 2026-02-20T21:46:21.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 538631168 bytes 2026-02-20T21:46:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 638341120 bytes 2026-02-20T21:46:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 684839936 bytes 2026-02-20T21:46:21.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 713401344 bytes 2026-02-20T21:46:21.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 735512576 bytes 2026-02-20T21:46:21.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 760030208 bytes 2026-02-20T21:46:21.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 784749568 bytes 2026-02-20T21:46:21.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 846100480 bytes 2026-02-20T21:46:21.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 873031680 bytes 2026-02-20T21:46:21.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 891163648 bytes 2026-02-20T21:46:22.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 995888128 bytes 2026-02-20T21:46:22.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1039534080 bytes 2026-02-20T21:46:22.111 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9174901082434939) > weight (0.25) 2026-02-20T21:46:22.111 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.726125230056426) > weight (0.25) 2026-02-20T21:46:22.112 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9983937942619858) > weight (0.25) 2026-02-20T21:46:22.112 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6219801864992608) > weight (0.25) 2026-02-20T21:46:22.112 INFO:tasks.rbd_mirror_thrash:waiting for 61.55598252295647 secs before thrashing 2026-02-20T21:46:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1105651712 bytes 2026-02-20T21:46:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1133388800 bytes 2026-02-20T21:46:22.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1158348800 bytes 2026-02-20T21:46:22.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1261404160 bytes 2026-02-20T21:46:22.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1305826304 bytes 2026-02-20T21:46:22.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1372051456 bytes 2026-02-20T21:46:22.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1404540928 bytes 2026-02-20T21:46:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1475982336 bytes 2026-02-20T21:46:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1537186816 bytes 2026-02-20T21:46:22.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1551605760 bytes 2026-02-20T21:46:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1599873024 bytes 2026-02-20T21:46:22.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1609947136 bytes 2026-02-20T21:46:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1646580736 bytes 2026-02-20T21:46:22.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1669486592 bytes 2026-02-20T21:46:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1714407424 bytes 2026-02-20T21:46:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1736411136 bytes 2026-02-20T21:46:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1802636288 bytes 2026-02-20T21:46:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1828293632 bytes 2026-02-20T21:46:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1871915008 bytes 2026-02-20T21:46:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1925956608 bytes 2026-02-20T21:46:23.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1948077056 bytes 2026-02-20T21:46:23.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1996572672 bytes 2026-02-20T21:46:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2052447232 bytes 2026-02-20T21:46:23.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2070301696 bytes 2026-02-20T21:46:23.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2100210688 bytes 2026-02-20T21:46:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2141972480 bytes 2026-02-20T21:46:23.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2173780992 bytes 2026-02-20T21:46:23.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2182284288 bytes 2026-02-20T21:46:23.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2251430912 bytes 2026-02-20T21:46:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2283342848 bytes 2026-02-20T21:46:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2360748032 bytes 2026-02-20T21:46:23.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2400934912 bytes 2026-02-20T21:46:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2418291712 bytes 2026-02-20T21:46:23.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2455260160 bytes 2026-02-20T21:46:23.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2517293056 bytes 2026-02-20T21:46:23.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2653481984 bytes 2026-02-20T21:46:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2678132736 bytes 2026-02-20T21:46:23.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2707446784 bytes 2026-02-20T21:46:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2723247104 bytes 2026-02-20T21:46:23.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2782765056 bytes 2026-02-20T21:46:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2793773056 bytes 2026-02-20T21:46:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2902718464 bytes 2026-02-20T21:46:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2913776640 bytes 2026-02-20T21:46:24.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2952611840 bytes 2026-02-20T21:46:24.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3031797760 bytes 2026-02-20T21:46:24.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3080487936 bytes 2026-02-20T21:46:24.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3191239680 bytes 2026-02-20T21:46:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3240714240 bytes 2026-02-20T21:46:24.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3254113280 bytes 2026-02-20T21:46:24.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3270658048 bytes 2026-02-20T21:46:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3277466624 bytes 2026-02-20T21:46:24.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3315843072 bytes 2026-02-20T21:46:24.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3316416512 bytes 2026-02-20T21:46:24.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3376057344 bytes 2026-02-20T21:46:24.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3408811008 bytes 2026-02-20T21:46:24.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3444208640 bytes 2026-02-20T21:46:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3499984896 bytes 2026-02-20T21:46:24.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3519316992 bytes 2026-02-20T21:46:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3523770368 bytes 2026-02-20T21:46:24.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3528297472 bytes 2026-02-20T21:46:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3603293184 bytes 2026-02-20T21:46:24.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3653484544 bytes 2026-02-20T21:46:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3701064704 bytes 2026-02-20T21:46:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3721667584 bytes 2026-02-20T21:46:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3723393024 bytes 2026-02-20T21:46:25.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3741046784 bytes 2026-02-20T21:46:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3799708672 bytes 2026-02-20T21:46:25.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3800329216 bytes 2026-02-20T21:46:25.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3874734080 bytes 2026-02-20T21:46:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3951738880 bytes 2026-02-20T21:46:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3985711104 bytes 2026-02-20T21:46:25.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3987548160 bytes 2026-02-20T21:46:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4035865600 bytes 2026-02-20T21:46:25.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4052620288 bytes 2026-02-20T21:46:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4109954048 bytes 2026-02-20T21:46:25.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4157579264 bytes 2026-02-20T21:46:25.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4161103872 bytes 2026-02-20T21:46:25.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4195580928 bytes 2026-02-20T21:46:25.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4240010240 bytes 2026-02-20T21:46:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4282522624 bytes 2026-02-20T21:46:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4304533504 bytes 2026-02-20T21:46:25.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4370567168 bytes 2026-02-20T21:46:25.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4397006848 bytes 2026-02-20T21:46:25.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4419535872 bytes 2026-02-20T21:46:25.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4508825600 bytes 2026-02-20T21:46:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4532131840 bytes 2026-02-20T21:46:26.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4560669696 bytes 2026-02-20T21:46:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4576678912 bytes 2026-02-20T21:46:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4598883328 bytes 2026-02-20T21:46:26.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4613231616 bytes 2026-02-20T21:46:26.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4660911104 bytes 2026-02-20T21:46:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4734280704 bytes 2026-02-20T21:46:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4779718656 bytes 2026-02-20T21:46:26.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4844250112 bytes 2026-02-20T21:46:26.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4903072768 bytes 2026-02-20T21:46:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4924054528 bytes 2026-02-20T21:46:26.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4948297728 bytes 2026-02-20T21:46:26.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 4964983808 bytes 2026-02-20T21:46:26.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 4998475776 bytes 2026-02-20T21:46:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5030282240 bytes 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:46:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:46:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:46:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:46:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local description_pattern= 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local service_pattern= 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:46:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:46:27.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='test: 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: global_id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+replaying 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: replaying, {"bytes_per_second":13142152.53,"bytes_per_snapshot":534880256.0,"last_snapshot_bytes":532939776,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623973,"remote_snapshot_timestamp":1771623973,"replay_state":"idle"} 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: service: mirror.3 on trial107 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:46:19 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: peer_sites: 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: name: cluster2 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: state: up+stopped 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: description: local image is primary 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: last_update: 2026-02-20 21:46:01' 2026-02-20T21:46:27.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:46:27.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: state: up+replaying 2026-02-20T21:46:27.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^ description: .*' 2026-02-20T21:46:27.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: description: replaying, {"bytes_per_second":13142152.53,"bytes_per_snapshot":534880256.0,"last_snapshot_bytes":532939776,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623973,"remote_snapshot_timestamp":1771623973,"replay_state":"idle"} 2026-02-20T21:46:27.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:46:27.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:46:27.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep '^up+' 2026-02-20T21:46:27.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:46:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:46:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: service: mirror.3 on trial107 2026-02-20T21:46:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update 2026-02-20T21:46:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:46:27.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:46:19' 2026-02-20T21:46:27.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:46:19' 2026-02-20T21:46:27.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:27.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:27.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:27.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state_pattern=up+replaying 2026-02-20T21:46:27.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local 'prev_last_update=2026-02-20 21:46:19' 2026-02-20T21:46:27.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status 2026-02-20T21:46:27.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ CEPH_ARGS= 2026-02-20T21:46:27.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:46:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status='OKOKOK142154266mirror.1trial10720.2.0-677-gf7870454falseOK42234272mirror.3trial10720.2.0-677-gf7870454trueOK43894391mirror.2trial10720.2.0-677-gf7870454falseOK44454448mirror.0trial10720.2.0-677-gf7870454falseOKtest4576ec0b-2750-4400-80ac-2ccee0758d15up+replayingreplaying, {"bytes_per_second":13142152.53,"bytes_per_snapshot":534880256.0,"last_snapshot_bytes":532939776,"last_snapshot_sync_seconds":0,"local_snapshot_timestamp":1771623973,"remote_snapshot_timestamp":1771623973,"replay_state":"idle"}42234272mirror.3trial1072026-02-20 21:46:19cluster2aa40d3f4-0618-4c9d-a0d0-5b630053911dup+stoppedlocal image is primary2026-02-20 21:46:01' 2026-02-20T21:46:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_update state 2026-02-20T21:46:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:46:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ last_update='2026-02-20 21:46:19' 2026-02-20T21:46:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:46:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=up+replaying 2026-02-20T21:46:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo up+replaying 2026-02-20T21:46:27.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep up+replaying 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:up+replaying 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap10 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap10 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap10 2026-02-20T21:46:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:46:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:46:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:46:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:46:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:46:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:46:57.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:46:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:46:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:46:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:46:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:46:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:46:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:46:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:46:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:46:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:46:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:46:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:46:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:46:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:46:58.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:46:58.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:46:58.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:58.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:46:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:46:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:46:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:46:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:46:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:46:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:46:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:46:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:46:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:46:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:46:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:46:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:46:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:46:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:46:58.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:58.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:46:58.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:46:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:46:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:46:58.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:46:58.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:46:58.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:46:58.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:46:58.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:46:58.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:46:58.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:46:58.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:46:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:46:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:46:58.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:46:58.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:46:58.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:46:58.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:46:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:46:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:46:58.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:46:58.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:46:58.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:46:58.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:46:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:46:58.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:46:58.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:46:58.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:46:58.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:46:58.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:46:58.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:46:58.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:46:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:46:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:46:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:46:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:46:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:46:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:46:58.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:47:00.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 32 2026-02-20T21:47:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:47:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:47:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:47:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:47:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:47:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:47:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:47:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:47:00.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:47:00.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:47:00.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:47:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:47:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:47:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:47:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:47:00.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:47:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=32 2026-02-20T21:47:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:47:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:47:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:47:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:47:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:47:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:47:00.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:47:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:47:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:47:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:47:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:47:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 32 = '' 2026-02-20T21:47:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:47:00.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:47:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:47:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:47:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:47:01.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:47:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:47:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:47:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:47:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:47:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 32 = '' 2026-02-20T21:47:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:47:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:47:02.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:02.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:02.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:02.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:02.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:02.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:47:02.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:47:02.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:47:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=32 2026-02-20T21:47:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 32 = 32 2026-02-20T21:47:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2026-02-20T21:47:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap10 2026-02-20T21:47:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:47:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:47:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:47:03.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2026-02-20T21:47:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:47:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:47:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:47:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 20 2026-02-20T21:47:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:47:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:45:31 2026 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:47:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:47:03.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:03.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snap9-clean ']' 2026-02-20T21:47:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-20T21:47:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap9-clean 2026-02-20T21:47:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:03.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:04.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap10 2026-02-20T21:47:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap10 2026-02-20T21:47:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:05.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:05.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ clean_snap_name=snap10-clean 2026-02-20T21:47:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror test snap10-clean 2026-02-20T21:47:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap10-clean 2026-02-20T21:47:05.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:47:06.089 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5759696433953142) > weight (0.25) 2026-02-20T21:47:06.089 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9884294432582984) > weight (0.25) 2026-02-20T21:47:06.089 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-20T21:47:06.090 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-20T21:47:06.090 INFO:tasks.rbd_mirror_thrash:waiting for 0.38397834210033843 secs before reviving daemons 2026-02-20T21:47:06.090 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T21:47:06.088+0000 7f6597136640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 19462) UID: 1000 2026-02-20T21:47:06.474 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-20T21:47:06.475 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:47:06.475 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-20T21:47:06.475 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T21:47:06.475 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T21:47:06.476 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T21:47:06.476 INFO:tasks.rbd_mirror_thrash:waiting for 103.30183367144244 secs before thrashing 2026-02-20T21:47:06.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:47:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:47:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:47:07.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:47:07.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:07.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:07.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:07.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:47:07.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:47:07.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:47:07.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:47:07.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:47:07.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:47:07.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:47:07.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:47:07.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:47:07.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:47:07.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:47:07.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:47:07.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:47:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:47:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:47:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:47:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:47:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:47:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:47:07.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:47:07.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:47:07.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:47:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:47:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:47:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:47:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:47:07.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:47:07.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:47:07.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:47:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:47:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:47:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:47:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:47:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:47:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:47:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:47:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:47:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:47:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:47:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:47:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:47:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:47:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:47:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:47:07.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:47:07.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:47:07.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:47:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:47:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:47:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:47:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:47:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:47:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:47:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:47:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:47:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:47:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:47:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:47:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:47:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:47:07.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:47:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:47:07.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:47:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:47:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:47:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:47:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:47:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:47:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:47:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:47:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:47:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:47:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:47:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:47:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:47:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:47:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:47:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:47:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:47:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:47:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:47:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:47:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:47:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:47:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:47:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:47:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror test 2026-02-20T21:47:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:07.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/test 2026-02-20T21:47:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 34 2026-02-20T21:47:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:47:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:47:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:47:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:47:09.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-test.status 2026-02-20T21:47:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-test.status 2026-02-20T21:47:09.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:47:09.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:47:09.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-test.status 2026-02-20T21:47:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-test.status 2026-02-20T21:47:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:47:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror test /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:47:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:09.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.status 2026-02-20T21:47:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:47:09.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/test --format xml 2026-02-20T21:47:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:47:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=34 2026-02-20T21:47:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:47:09.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:47:09.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:47:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror test /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.status 2026-02-20T21:47:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/test --format xml 2026-02-20T21:47:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:47:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:47:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=34 2026-02-20T21:47:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 34 = 34 2026-02-20T21:47:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2026-02-20T21:47:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap10-clean 2026-02-20T21:47:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:47:09.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:47:09.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:47:10.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2026-02-20T21:47:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'test': 2026-02-20T21:47:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T21:47:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:47:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 21 2026-02-20T21:47:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10a841b7487b 2026-02-20T21:47:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10a841b7487b 2026-02-20T21:47:10.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:47:03 2026 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:38:51 2026 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4576ec0b-2750-4400-80ac-2ccee0758d15 2026-02-20T21:47:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:47:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:10.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 10 2026-02-20T21:47:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap1 2026-02-20T21:47:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap1 2026-02-20T21:47:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap1 2026-02-20T21:47:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:12.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:12.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap1-clean 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap1-clean 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:12.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:13.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:13.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:13.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap2 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap2 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap2 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:13.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:14.225 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:14.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:15.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap2-clean 2026-02-20T21:47:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-20T21:47:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap2-clean 2026-02-20T21:47:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:15.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:16.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:16.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap3 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap3 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap3 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:17.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:17.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap3-clean 2026-02-20T21:47:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-20T21:47:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap3-clean 2026-02-20T21:47:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:18.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:18.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:18.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:19.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap4 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap4 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap4 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:19.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:19.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:20.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:20.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:20.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:20.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap4-clean 2026-02-20T21:47:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-20T21:47:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap4-clean 2026-02-20T21:47:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:20.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:20.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:20.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:21.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:21.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:21.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap5 2026-02-20T21:47:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap5 2026-02-20T21:47:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap5 2026-02-20T21:47:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:21.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap5-clean 2026-02-20T21:47:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-20T21:47:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap5-clean 2026-02-20T21:47:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:23.668 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.3991467446791176) > weight (0.25) 2026-02-20T21:47:23.669 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-20T21:47:23.669 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-20T21:47:23.669 INFO:tasks.rbd_mirror_thrash:waiting for 6.879238773391366 secs before reviving daemons 2026-02-20T21:47:23.670 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T21:47:23.663+0000 7f15d13ac640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 19338) UID: 1000 2026-02-20T21:47:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:24.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:24.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap6 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap6 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap6 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:25.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:25.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap6-clean 2026-02-20T21:47:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-20T21:47:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap6-clean 2026-02-20T21:47:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:26.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:27.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:27.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap7 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap7 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap7 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:27.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:28.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:28.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap7-clean 2026-02-20T21:47:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-20T21:47:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:28.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap7-clean 2026-02-20T21:47:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:28.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:29.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:30.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap8 2026-02-20T21:47:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap8 2026-02-20T21:47:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap8 2026-02-20T21:47:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:30.549 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-20T21:47:30.549 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:47:30.550 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-20T21:47:30.550 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T21:47:30.550 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T21:47:30.553 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T21:47:30.553 INFO:tasks.rbd_mirror_thrash:waiting for 68.17459477952283 secs before thrashing 2026-02-20T21:47:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:31.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap8-clean 2026-02-20T21:47:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-20T21:47:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap8-clean 2026-02-20T21:47:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:31.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:31.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:31.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:31.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:32.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:32.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:32.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap9 2026-02-20T21:47:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap9 2026-02-20T21:47:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap9 2026-02-20T21:47:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:33.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:33.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:34.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap9-clean 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap9-clean 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:34.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:35.274 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap10 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap10 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap10 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:36.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:36.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:36.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap10-clean 2026-02-20T21:47:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror test snap10-clean 2026-02-20T21:47:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap10-clean 2026-02-20T21:47:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:47:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:37.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export 2026-02-20T21:47:37.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:37.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:47:38.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:38.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-test.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-test.export 2026-02-20T21:47:38.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:38.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 10 2026-02-20T21:47:38.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:38.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap1 2026-02-20T21:47:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap1 2026-02-20T21:47:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap1 2026-02-20T21:47:38.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2026-02-20T21:47:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap1-clean 2026-02-20T21:47:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2026-02-20T21:47:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap1-clean 2026-02-20T21:47:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2026-02-20T21:47:39.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:39.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap2 2026-02-20T21:47:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap2 2026-02-20T21:47:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap2 2026-02-20T21:47:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2026-02-20T21:47:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:40.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap2-clean 2026-02-20T21:47:40.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2026-02-20T21:47:40.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:40.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:40.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:40.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap2-clean 2026-02-20T21:47:40.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2026-02-20T21:47:41.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap3 2026-02-20T21:47:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap3 2026-02-20T21:47:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:41.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap3 2026-02-20T21:47:41.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2026-02-20T21:47:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap3-clean 2026-02-20T21:47:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2026-02-20T21:47:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap3-clean 2026-02-20T21:47:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2026-02-20T21:47:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap4 2026-02-20T21:47:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap4 2026-02-20T21:47:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap4 2026-02-20T21:47:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2026-02-20T21:47:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:44.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap4-clean 2026-02-20T21:47:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2026-02-20T21:47:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap4-clean 2026-02-20T21:47:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2026-02-20T21:47:45.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap5 2026-02-20T21:47:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap5 2026-02-20T21:47:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap5 2026-02-20T21:47:45.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2026-02-20T21:47:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap5-clean 2026-02-20T21:47:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2026-02-20T21:47:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap5-clean 2026-02-20T21:47:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2026-02-20T21:47:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap6 2026-02-20T21:47:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap6 2026-02-20T21:47:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:47.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:47.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap6 2026-02-20T21:47:47.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2026-02-20T21:47:48.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap6-clean 2026-02-20T21:47:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2026-02-20T21:47:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap6-clean 2026-02-20T21:47:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2026-02-20T21:47:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap7 2026-02-20T21:47:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap7 2026-02-20T21:47:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:49.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:49.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:49.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap7 2026-02-20T21:47:49.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2026-02-20T21:47:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:50.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap7-clean 2026-02-20T21:47:50.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2026-02-20T21:47:50.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:50.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:50.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:50.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap7-clean 2026-02-20T21:47:50.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2026-02-20T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap8 2026-02-20T21:47:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap8 2026-02-20T21:47:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap8 2026-02-20T21:47:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2026-02-20T21:47:52.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap8-clean 2026-02-20T21:47:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2026-02-20T21:47:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap8-clean 2026-02-20T21:47:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2026-02-20T21:47:53.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:53.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:53.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap9 2026-02-20T21:47:53.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap9 2026-02-20T21:47:53.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:53.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:53.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:53.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap9 2026-02-20T21:47:53.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2026-02-20T21:47:54.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap9-clean 2026-02-20T21:47:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2026-02-20T21:47:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap9-clean 2026-02-20T21:47:54.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2026-02-20T21:47:55.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T21:47:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap10 2026-02-20T21:47:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap10 2026-02-20T21:47:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap10 2026-02-20T21:47:55.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2026-02-20T21:47:56.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap10-clean 2026-02-20T21:47:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2026-02-20T21:47:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap10-clean 2026-02-20T21:47:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2026-02-20T21:47:57.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T21:47:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror test 2026-02-20T21:47:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:47:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T21:47:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror test 2026-02-20T21:47:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:47:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:57.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:57.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-20T21:47:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-20T21:47:57.746 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T21:47:57.739+0000 7ffab2f5f640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x557fa69f1ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T21:47:57.746 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T21:47:57.739+0000 7ffab8f6b640 -1 rbd::mirror::ImageReplayer: 0x557fa68f1b80 [3/4576ec0b-2750-4400-80ac-2ccee0758d15] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T21:47:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:47:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:47:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-20T21:47:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:47:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:47:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:47:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T21:47:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T21:47:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:47:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T21:47:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:47:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:47:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T21:47:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=test 2026-02-20T21:47:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:47:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:48:00.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:48:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T21:48:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:48:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T21:48:00.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-02-20T21:48:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:48:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=test 2026-02-20T21:48:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T21:48:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T21:48:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T21:48:00.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T21:48:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:48:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:48:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T21:48:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=test 2026-02-20T21:48:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:48:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:48:00.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:48:00.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T21:48:00.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T21:48:00.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T21:48:00.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:00.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ testlog 'TEST: create many images' 2026-02-20T21:48:00.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ tee -a /tmp/tmp.2SC0OFePO0/rbd-mirror.test.log 2026-02-20T21:48:00.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ date '+%F %T' 2026-02-20T21:48:00.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo 2026-02-20 21:48:00 TEST: create many images 2026-02-20T21:48:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:2026-02-20 21:48:00 TEST: create many images 2026-02-20T21:48:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ snap_name=snap 2026-02-20T21:48:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 50 2026-02-20T21:48:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:48:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_1 2026-02-20T21:48:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 snapshot 128M 2026-02-20T21:48:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:00.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_1 128M 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:00.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:48:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:48:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:48:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:48:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:00.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_1 2026-02-20T21:48:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:48:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:48:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_1 snapshot 2026-02-20T21:48:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:48:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:00.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2026-02-20T21:48:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:48:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-02-20T21:48:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_1': 2026-02-20T21:48:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:48:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:48:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:48:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 127376f64fa2 2026-02-20T21:48:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.127376f64fa2 2026-02-20T21:48:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:48:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:48:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:48:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:48:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:00 2026 2026-02-20T21:48:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:00 2026 2026-02-20T21:48:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:00 2026 2026-02-20T21:48:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:48:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:48:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: db87f39c-9375-4487-aa96-a4de9e1499de 2026-02-20T21:48:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:48:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:48:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-20T21:48:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:48:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:48:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=14 2026-02-20T21:48:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:48:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:48:00.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 57741312 bytes 2026-02-20T21:48:00.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 100457472 bytes 2026-02-20T21:48:00.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 192159744 bytes 2026-02-20T21:48:01.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 237086720 bytes 2026-02-20T21:48:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 251329536 bytes 2026-02-20T21:48:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 278065152 bytes 2026-02-20T21:48:01.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 343775232 bytes 2026-02-20T21:48:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 368030720 bytes 2026-02-20T21:48:01.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 435843072 bytes 2026-02-20T21:48:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 491904000 bytes 2026-02-20T21:48:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 553389056 bytes 2026-02-20T21:48:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 613950464 bytes 2026-02-20T21:48:01.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 628114432 bytes 2026-02-20T21:48:01.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 683626496 bytes 2026-02-20T21:48:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 719572992 bytes 2026-02-20T21:48:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 773571584 bytes 2026-02-20T21:48:01.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 823188480 bytes 2026-02-20T21:48:01.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 852102144 bytes 2026-02-20T21:48:01.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 905055232 bytes 2026-02-20T21:48:01.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 967966720 bytes 2026-02-20T21:48:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1014253568 bytes 2026-02-20T21:48:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1077920768 bytes 2026-02-20T21:48:01.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1167753216 bytes 2026-02-20T21:48:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1212364800 bytes 2026-02-20T21:48:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1250235392 bytes 2026-02-20T21:48:02.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1286717440 bytes 2026-02-20T21:48:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1309914112 bytes 2026-02-20T21:48:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1345067008 bytes 2026-02-20T21:48:02.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1370729472 bytes 2026-02-20T21:48:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1384178688 bytes 2026-02-20T21:48:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1427301376 bytes 2026-02-20T21:48:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1490491392 bytes 2026-02-20T21:48:02.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1524723712 bytes 2026-02-20T21:48:02.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1575596032 bytes 2026-02-20T21:48:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1623707648 bytes 2026-02-20T21:48:02.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1662417920 bytes 2026-02-20T21:48:02.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1689596928 bytes 2026-02-20T21:48:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1790861312 bytes 2026-02-20T21:48:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1834960896 bytes 2026-02-20T21:48:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1890541568 bytes 2026-02-20T21:48:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1920405504 bytes 2026-02-20T21:48:02.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1966876672 bytes 2026-02-20T21:48:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2063334400 bytes 2026-02-20T21:48:02.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2114906112 bytes 2026-02-20T21:48:02.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2136176640 bytes 2026-02-20T21:48:02.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2277285888 bytes 2026-02-20T21:48:02.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2320754688 bytes 2026-02-20T21:48:02.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2328402944 bytes 2026-02-20T21:48:03.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2355870720 bytes 2026-02-20T21:48:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2418234368 bytes 2026-02-20T21:48:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2428698624 bytes 2026-02-20T21:48:03.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2433890304 bytes 2026-02-20T21:48:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2434500608 bytes 2026-02-20T21:48:03.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2470918144 bytes 2026-02-20T21:48:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2544566272 bytes 2026-02-20T21:48:03.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2562524160 bytes 2026-02-20T21:48:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2601873408 bytes 2026-02-20T21:48:03.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2619383808 bytes 2026-02-20T21:48:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2657691648 bytes 2026-02-20T21:48:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2685136896 bytes 2026-02-20T21:48:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2729206784 bytes 2026-02-20T21:48:03.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2753072128 bytes 2026-02-20T21:48:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2813415424 bytes 2026-02-20T21:48:03.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2847566848 bytes 2026-02-20T21:48:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2863068160 bytes 2026-02-20T21:48:03.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2902452224 bytes 2026-02-20T21:48:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2926579712 bytes 2026-02-20T21:48:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2956587008 bytes 2026-02-20T21:48:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2976332800 bytes 2026-02-20T21:48:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3003587584 bytes 2026-02-20T21:48:03.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3011389440 bytes 2026-02-20T21:48:03.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3026543616 bytes 2026-02-20T21:48:03.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3063611392 bytes 2026-02-20T21:48:03.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3092671488 bytes 2026-02-20T21:48:03.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3131838464 bytes 2026-02-20T21:48:04.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3167331328 bytes 2026-02-20T21:48:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3200077824 bytes 2026-02-20T21:48:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3216174080 bytes 2026-02-20T21:48:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3272690688 bytes 2026-02-20T21:48:04.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3275906048 bytes 2026-02-20T21:48:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3332682752 bytes 2026-02-20T21:48:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3388485632 bytes 2026-02-20T21:48:04.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3465292800 bytes 2026-02-20T21:48:04.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3525343232 bytes 2026-02-20T21:48:04.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3599666176 bytes 2026-02-20T21:48:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3732670464 bytes 2026-02-20T21:48:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3759710208 bytes 2026-02-20T21:48:04.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3779459072 bytes 2026-02-20T21:48:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3816013824 bytes 2026-02-20T21:48:04.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3887055872 bytes 2026-02-20T21:48:04.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3907747840 bytes 2026-02-20T21:48:04.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3976550400 bytes 2026-02-20T21:48:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4001845248 bytes 2026-02-20T21:48:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4043420672 bytes 2026-02-20T21:48:04.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4047443968 bytes 2026-02-20T21:48:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4103307264 bytes 2026-02-20T21:48:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4197413888 bytes 2026-02-20T21:48:05.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4198022144 bytes 2026-02-20T21:48:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4259393536 bytes 2026-02-20T21:48:05.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4286344192 bytes 2026-02-20T21:48:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4319777792 bytes 2026-02-20T21:48:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4400258048 bytes 2026-02-20T21:48:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4443121664 bytes 2026-02-20T21:48:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4467960832 bytes 2026-02-20T21:48:05.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4502155264 bytes 2026-02-20T21:48:05.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4544665600 bytes 2026-02-20T21:48:05.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4583557120 bytes 2026-02-20T21:48:05.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4620460032 bytes 2026-02-20T21:48:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4694569984 bytes 2026-02-20T21:48:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4732347392 bytes 2026-02-20T21:48:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4773881856 bytes 2026-02-20T21:48:05.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4826756096 bytes 2026-02-20T21:48:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4893376512 bytes 2026-02-20T21:48:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4916201472 bytes 2026-02-20T21:48:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4945200128 bytes 2026-02-20T21:48:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4997144576 bytes 2026-02-20T21:48:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5019773952 bytes 2026-02-20T21:48:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5028581376 bytes 2026-02-20T21:48:05.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5069718528 bytes 2026-02-20T21:48:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5158924288 bytes 2026-02-20T21:48:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5177872384 bytes 2026-02-20T21:48:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5195817984 bytes 2026-02-20T21:48:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5209254912 bytes 2026-02-20T21:48:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5235713024 bytes 2026-02-20T21:48:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5283044352 bytes 2026-02-20T21:48:06.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5347317760 bytes 2026-02-20T21:48:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5349570560 bytes 2026-02-20T21:48:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5377026048 bytes 2026-02-20T21:48:06.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5428304896 bytes 2026-02-20T21:48:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5488939008 bytes 2026-02-20T21:48:06.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5536900096 bytes 2026-02-20T21:48:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5562412032 bytes 2026-02-20T21:48:06.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5640763392 bytes 2026-02-20T21:48:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5697369088 bytes 2026-02-20T21:48:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5753084928 bytes 2026-02-20T21:48:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5768022016 bytes 2026-02-20T21:48:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5790481408 bytes 2026-02-20T21:48:06.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5876468736 bytes 2026-02-20T21:48:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5914345472 bytes 2026-02-20T21:48:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5983202304 bytes 2026-02-20T21:48:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6050751488 bytes 2026-02-20T21:48:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6100813824 bytes 2026-02-20T21:48:06.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6140141568 bytes 2026-02-20T21:48:07.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6178607104 bytes 2026-02-20T21:48:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6187374592 bytes 2026-02-20T21:48:07.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6227623936 bytes 2026-02-20T21:48:07.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6272407552 bytes 2026-02-20T21:48:07.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6297242624 bytes 2026-02-20T21:48:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6306363392 bytes 2026-02-20T21:48:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6339733504 bytes 2026-02-20T21:48:07.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6361878528 bytes 2026-02-20T21:48:07.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6458396672 bytes 2026-02-20T21:48:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6500973568 bytes 2026-02-20T21:48:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6523234304 bytes 2026-02-20T21:48:07.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6577247232 bytes 2026-02-20T21:48:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6592037888 bytes 2026-02-20T21:48:07.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6596838400 bytes 2026-02-20T21:48:07.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6647456768 bytes 2026-02-20T21:48:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6709037056 bytes 2026-02-20T21:48:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6732115968 bytes 2026-02-20T21:48:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6755966976 bytes 2026-02-20T21:48:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6794051584 bytes 2026-02-20T21:48:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6823984128 bytes 2026-02-20T21:48:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6843708416 bytes 2026-02-20T21:48:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6885300224 bytes 2026-02-20T21:48:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6904929280 bytes 2026-02-20T21:48:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6931405824 bytes 2026-02-20T21:48:07.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6991556608 bytes 2026-02-20T21:48:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7066731520 bytes 2026-02-20T21:48:08.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7122337792 bytes 2026-02-20T21:48:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7203906560 bytes 2026-02-20T21:48:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7269395456 bytes 2026-02-20T21:48:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7297222656 bytes 2026-02-20T21:48:08.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7375342592 bytes 2026-02-20T21:48:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7443903488 bytes 2026-02-20T21:48:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7470430208 bytes 2026-02-20T21:48:08.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7522480128 bytes 2026-02-20T21:48:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7557374976 bytes 2026-02-20T21:48:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7595837440 bytes 2026-02-20T21:48:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7654010880 bytes 2026-02-20T21:48:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7722391552 bytes 2026-02-20T21:48:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7728957440 bytes 2026-02-20T21:48:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7732154368 bytes 2026-02-20T21:48:08.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7763618816 bytes 2026-02-20T21:48:08.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7799674880 bytes 2026-02-20T21:48:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7840676864 bytes 2026-02-20T21:48:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7867675648 bytes 2026-02-20T21:48:08.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7907361792 bytes 2026-02-20T21:48:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7966344192 bytes 2026-02-20T21:48:08.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8037098496 bytes 2026-02-20T21:48:09.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8135464960 bytes 2026-02-20T21:48:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8136055808 bytes 2026-02-20T21:48:09.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8136643584 bytes 2026-02-20T21:48:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8155488256 bytes 2026-02-20T21:48:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8181377024 bytes 2026-02-20T21:48:09.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8289837056 bytes 2026-02-20T21:48:09.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8314756096 bytes 2026-02-20T21:48:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8328970240 bytes 2026-02-20T21:48:09.340 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8355515392 bytes 2026-02-20T21:48:09.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8379102208 bytes 2026-02-20T21:48:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8414202880 bytes 2026-02-20T21:48:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8441421824 bytes 2026-02-20T21:48:09.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8526814208 bytes 2026-02-20T21:48:09.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8542218240 bytes 2026-02-20T21:48:09.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8579545088 bytes 2026-02-20T21:48:09.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8587848704 bytes 2026-02-20T21:48:09.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8632340480 bytes 2026-02-20T21:48:09.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8642959360 bytes 2026-02-20T21:48:09.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8643549184 bytes 2026-02-20T21:48:09.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8661391360 bytes 2026-02-20T21:48:09.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8662012928 bytes 2026-02-20T21:48:09.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8700327936 bytes 2026-02-20T21:48:09.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8772529152 bytes 2026-02-20T21:48:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8860969984 bytes 2026-02-20T21:48:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8872209408 bytes 2026-02-20T21:48:10.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 8893558784 bytes 2026-02-20T21:48:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 8923840512 bytes 2026-02-20T21:48:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 8953448448 bytes 2026-02-20T21:48:10.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 8980599808 bytes 2026-02-20T21:48:10.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9050436608 bytes 2026-02-20T21:48:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9161916416 bytes 2026-02-20T21:48:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9162527744 bytes 2026-02-20T21:48:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9231241216 bytes 2026-02-20T21:48:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9256939520 bytes 2026-02-20T21:48:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9365641216 bytes 2026-02-20T21:48:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9376318464 bytes 2026-02-20T21:48:10.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9399000064 bytes 2026-02-20T21:48:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9429518336 bytes 2026-02-20T21:48:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9471536128 bytes 2026-02-20T21:48:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9521812480 bytes 2026-02-20T21:48:10.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9545064448 bytes 2026-02-20T21:48:10.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9568698368 bytes 2026-02-20T21:48:10.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9590460416 bytes 2026-02-20T21:48:10.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9619043328 bytes 2026-02-20T21:48:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9651621888 bytes 2026-02-20T21:48:10.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 9720664064 bytes 2026-02-20T21:48:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 9803297792 bytes 2026-02-20T21:48:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 9811303424 bytes 2026-02-20T21:48:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 9918508032 bytes 2026-02-20T21:48:11.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 9931969536 bytes 2026-02-20T21:48:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 9981564928 bytes 2026-02-20T21:48:11.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10026250240 bytes 2026-02-20T21:48:11.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 10093624320 bytes 2026-02-20T21:48:11.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 10115116032 bytes 2026-02-20T21:48:11.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10140158976 bytes 2026-02-20T21:48:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10236300288 bytes 2026-02-20T21:48:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10284169216 bytes 2026-02-20T21:48:11.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10321854464 bytes 2026-02-20T21:48:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 10349099008 bytes 2026-02-20T21:48:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 10375785472 bytes 2026-02-20T21:48:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25100 IOs, 10393317376 bytes 2026-02-20T21:48:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25200 IOs, 10476837888 bytes 2026-02-20T21:48:11.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25300 IOs, 10539830272 bytes 2026-02-20T21:48:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25400 IOs, 10561004544 bytes 2026-02-20T21:48:11.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25500 IOs, 10561607680 bytes 2026-02-20T21:48:11.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25600 IOs, 10642049024 bytes 2026-02-20T21:48:11.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25700 IOs, 10646509568 bytes 2026-02-20T21:48:11.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25800 IOs, 10681060352 bytes 2026-02-20T21:48:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25900 IOs, 10767386624 bytes 2026-02-20T21:48:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26000 IOs, 10800193536 bytes 2026-02-20T21:48:12.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26100 IOs, 10829292544 bytes 2026-02-20T21:48:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26200 IOs, 10931705856 bytes 2026-02-20T21:48:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26300 IOs, 10955633664 bytes 2026-02-20T21:48:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26400 IOs, 10979616768 bytes 2026-02-20T21:48:12.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26500 IOs, 11025470464 bytes 2026-02-20T21:48:12.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26600 IOs, 11040782336 bytes 2026-02-20T21:48:12.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26700 IOs, 11099555840 bytes 2026-02-20T21:48:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26800 IOs, 11115876352 bytes 2026-02-20T21:48:12.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26900 IOs, 11184528384 bytes 2026-02-20T21:48:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27000 IOs, 11201580032 bytes 2026-02-20T21:48:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27100 IOs, 11245584384 bytes 2026-02-20T21:48:12.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27200 IOs, 11314853888 bytes 2026-02-20T21:48:12.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27300 IOs, 11323554816 bytes 2026-02-20T21:48:12.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27400 IOs, 11365868544 bytes 2026-02-20T21:48:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27500 IOs, 11402054656 bytes 2026-02-20T21:48:12.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27600 IOs, 11412280320 bytes 2026-02-20T21:48:12.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27700 IOs, 11454473216 bytes 2026-02-20T21:48:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27800 IOs, 11482558464 bytes 2026-02-20T21:48:12.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27900 IOs, 11546240000 bytes 2026-02-20T21:48:12.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28000 IOs, 11632233472 bytes 2026-02-20T21:48:12.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28100 IOs, 11723399168 bytes 2026-02-20T21:48:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28200 IOs, 11728691200 bytes 2026-02-20T21:48:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28300 IOs, 11799264256 bytes 2026-02-20T21:48:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28400 IOs, 11842300928 bytes 2026-02-20T21:48:13.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28500 IOs, 11900416000 bytes 2026-02-20T21:48:13.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28600 IOs, 11925536768 bytes 2026-02-20T21:48:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28700 IOs, 11932257280 bytes 2026-02-20T21:48:13.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28800 IOs, 11984479232 bytes 2026-02-20T21:48:13.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28900 IOs, 12049602560 bytes 2026-02-20T21:48:13.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29000 IOs, 12079468544 bytes 2026-02-20T21:48:13.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29100 IOs, 12110046208 bytes 2026-02-20T21:48:13.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29200 IOs, 12229387264 bytes 2026-02-20T21:48:13.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29300 IOs, 12229977088 bytes 2026-02-20T21:48:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29400 IOs, 12260933632 bytes 2026-02-20T21:48:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29500 IOs, 12300753920 bytes 2026-02-20T21:48:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29600 IOs, 12367038464 bytes 2026-02-20T21:48:13.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29700 IOs, 12416287744 bytes 2026-02-20T21:48:13.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29800 IOs, 12467749888 bytes 2026-02-20T21:48:13.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29900 IOs, 12522811392 bytes 2026-02-20T21:48:13.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30000 IOs, 12592159744 bytes 2026-02-20T21:48:13.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30100 IOs, 12691373056 bytes 2026-02-20T21:48:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30200 IOs, 12770611200 bytes 2026-02-20T21:48:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30300 IOs, 12806651904 bytes 2026-02-20T21:48:14.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30400 IOs, 12835673088 bytes 2026-02-20T21:48:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30500 IOs, 12863550464 bytes 2026-02-20T21:48:14.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30600 IOs, 12866066432 bytes 2026-02-20T21:48:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30700 IOs, 12934316032 bytes 2026-02-20T21:48:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30800 IOs, 12955289600 bytes 2026-02-20T21:48:14.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:30900 IOs, 13011116032 bytes 2026-02-20T21:48:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31000 IOs, 13069021184 bytes 2026-02-20T21:48:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31100 IOs, 13120064512 bytes 2026-02-20T21:48:14.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31200 IOs, 13137394688 bytes 2026-02-20T21:48:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31300 IOs, 13186780160 bytes 2026-02-20T21:48:14.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31400 IOs, 13216483328 bytes 2026-02-20T21:48:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31500 IOs, 13294514176 bytes 2026-02-20T21:48:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31600 IOs, 13328621568 bytes 2026-02-20T21:48:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31700 IOs, 13388071936 bytes 2026-02-20T21:48:14.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31800 IOs, 13466541056 bytes 2026-02-20T21:48:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:31900 IOs, 13498293248 bytes 2026-02-20T21:48:14.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:32000 IOs, 13500884992 bytes 2026-02-20T21:48:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:48:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:48:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:48:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:48:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_2 2026-02-20T21:48:14.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 snapshot 128M 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_2 128M 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_2 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_2 snapshot 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2026-02-20T21:48:15.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:48:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_2': 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 127f1dc71d68 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.127f1dc71d68 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:14 2026 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:14 2026 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:14 2026 2026-02-20T21:48:15.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:48:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:48:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 6f61a2e4-1504-47d7-888d-5162115d91ce 2026-02-20T21:48:15.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:48:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:48:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-20T21:48:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:48:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:48:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:48:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:48:15.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:48:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 29625344 bytes 2026-02-20T21:48:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 66992128 bytes 2026-02-20T21:48:15.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 119256064 bytes 2026-02-20T21:48:15.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 159269888 bytes 2026-02-20T21:48:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 205713408 bytes 2026-02-20T21:48:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 206299136 bytes 2026-02-20T21:48:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 227857408 bytes 2026-02-20T21:48:15.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 264228864 bytes 2026-02-20T21:48:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 321635328 bytes 2026-02-20T21:48:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 367267840 bytes 2026-02-20T21:48:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 400105472 bytes 2026-02-20T21:48:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 434202624 bytes 2026-02-20T21:48:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 499112960 bytes 2026-02-20T21:48:16.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 542988288 bytes 2026-02-20T21:48:16.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 545829888 bytes 2026-02-20T21:48:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 568884224 bytes 2026-02-20T21:48:16.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 604192768 bytes 2026-02-20T21:48:16.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 655748096 bytes 2026-02-20T21:48:16.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 710870016 bytes 2026-02-20T21:48:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 760155136 bytes 2026-02-20T21:48:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 839227392 bytes 2026-02-20T21:48:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 851637248 bytes 2026-02-20T21:48:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 990678016 bytes 2026-02-20T21:48:16.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1065447424 bytes 2026-02-20T21:48:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1114745856 bytes 2026-02-20T21:48:16.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1116684288 bytes 2026-02-20T21:48:16.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1124350976 bytes 2026-02-20T21:48:16.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1147590656 bytes 2026-02-20T21:48:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1210866688 bytes 2026-02-20T21:48:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1247206400 bytes 2026-02-20T21:48:16.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1317334016 bytes 2026-02-20T21:48:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1355803648 bytes 2026-02-20T21:48:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1383069696 bytes 2026-02-20T21:48:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1401641984 bytes 2026-02-20T21:48:17.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1411551232 bytes 2026-02-20T21:48:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1436405760 bytes 2026-02-20T21:48:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1462341632 bytes 2026-02-20T21:48:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1506233344 bytes 2026-02-20T21:48:17.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1510626304 bytes 2026-02-20T21:48:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1544801280 bytes 2026-02-20T21:48:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1582544896 bytes 2026-02-20T21:48:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1614252032 bytes 2026-02-20T21:48:17.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1655706624 bytes 2026-02-20T21:48:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1717773312 bytes 2026-02-20T21:48:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1770881024 bytes 2026-02-20T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1817556992 bytes 2026-02-20T21:48:17.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1840541696 bytes 2026-02-20T21:48:17.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1841152000 bytes 2026-02-20T21:48:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1909237760 bytes 2026-02-20T21:48:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1987835904 bytes 2026-02-20T21:48:17.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2043957248 bytes 2026-02-20T21:48:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2044624896 bytes 2026-02-20T21:48:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2113309696 bytes 2026-02-20T21:48:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2191415296 bytes 2026-02-20T21:48:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2340967424 bytes 2026-02-20T21:48:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2371900416 bytes 2026-02-20T21:48:18.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2434754560 bytes 2026-02-20T21:48:18.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2474042368 bytes 2026-02-20T21:48:18.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2558120960 bytes 2026-02-20T21:48:18.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2686979072 bytes 2026-02-20T21:48:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2755369984 bytes 2026-02-20T21:48:18.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2800694272 bytes 2026-02-20T21:48:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2823143424 bytes 2026-02-20T21:48:18.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2885596160 bytes 2026-02-20T21:48:18.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2914334720 bytes 2026-02-20T21:48:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2930853888 bytes 2026-02-20T21:48:18.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2931468288 bytes 2026-02-20T21:48:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2943009792 bytes 2026-02-20T21:48:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3004075008 bytes 2026-02-20T21:48:18.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3049451520 bytes 2026-02-20T21:48:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3190247424 bytes 2026-02-20T21:48:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3258957824 bytes 2026-02-20T21:48:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3270068224 bytes 2026-02-20T21:48:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3326789632 bytes 2026-02-20T21:48:19.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3394578432 bytes 2026-02-20T21:48:19.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3395171328 bytes 2026-02-20T21:48:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3511665664 bytes 2026-02-20T21:48:19.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3535946752 bytes 2026-02-20T21:48:19.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3602244608 bytes 2026-02-20T21:48:19.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3666945024 bytes 2026-02-20T21:48:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3704341504 bytes 2026-02-20T21:48:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3704918016 bytes 2026-02-20T21:48:19.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3726736384 bytes 2026-02-20T21:48:19.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3751388160 bytes 2026-02-20T21:48:19.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3786405888 bytes 2026-02-20T21:48:19.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3854788608 bytes 2026-02-20T21:48:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3886904320 bytes 2026-02-20T21:48:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3894300672 bytes 2026-02-20T21:48:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3940992000 bytes 2026-02-20T21:48:20.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3987661824 bytes 2026-02-20T21:48:20.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3997904896 bytes 2026-02-20T21:48:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4043603968 bytes 2026-02-20T21:48:20.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4091133952 bytes 2026-02-20T21:48:20.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4171218944 bytes 2026-02-20T21:48:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4229415936 bytes 2026-02-20T21:48:20.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4300654592 bytes 2026-02-20T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4389615616 bytes 2026-02-20T21:48:20.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4468804608 bytes 2026-02-20T21:48:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4514461696 bytes 2026-02-20T21:48:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4515693568 bytes 2026-02-20T21:48:20.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4548819968 bytes 2026-02-20T21:48:20.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4593167360 bytes 2026-02-20T21:48:20.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4626171904 bytes 2026-02-20T21:48:20.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4675196928 bytes 2026-02-20T21:48:20.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4687395840 bytes 2026-02-20T21:48:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4722029568 bytes 2026-02-20T21:48:21.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4780848128 bytes 2026-02-20T21:48:21.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4814002176 bytes 2026-02-20T21:48:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4843513856 bytes 2026-02-20T21:48:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4868816896 bytes 2026-02-20T21:48:21.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4910299136 bytes 2026-02-20T21:48:21.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4969731072 bytes 2026-02-20T21:48:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4998325248 bytes 2026-02-20T21:48:21.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5025464320 bytes 2026-02-20T21:48:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5061224448 bytes 2026-02-20T21:48:21.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5113549824 bytes 2026-02-20T21:48:21.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_3 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 snapshot 128M 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:21.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_3 128M 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_3 2026-02-20T21:48:21.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:48:21.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:48:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_3 snapshot 2026-02-20T21:48:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:48:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:21.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2026-02-20T21:48:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:48:22.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-02-20T21:48:22.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_3': 2026-02-20T21:48:22.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:48:22.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:48:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:48:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 128b3b492fe4 2026-02-20T21:48:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.128b3b492fe4 2026-02-20T21:48:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:48:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:48:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:48:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:48:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:21 2026 2026-02-20T21:48:22.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:21 2026 2026-02-20T21:48:22.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:21 2026 2026-02-20T21:48:22.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:48:22.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:48:22.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: ad6926aa-1d75-40e4-80e4-b13fcd076244 2026-02-20T21:48:22.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:48:22.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:48:22.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-20T21:48:22.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:22.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:22.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:48:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:48:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T21:48:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:48:22.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:48:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 57214976 bytes 2026-02-20T21:48:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 92009472 bytes 2026-02-20T21:48:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 92633088 bytes 2026-02-20T21:48:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 140724224 bytes 2026-02-20T21:48:22.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 199399424 bytes 2026-02-20T21:48:22.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 260880384 bytes 2026-02-20T21:48:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 268869632 bytes 2026-02-20T21:48:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 345899008 bytes 2026-02-20T21:48:23.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 379636736 bytes 2026-02-20T21:48:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 449465344 bytes 2026-02-20T21:48:23.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 484436992 bytes 2026-02-20T21:48:23.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 509840384 bytes 2026-02-20T21:48:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 561185792 bytes 2026-02-20T21:48:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 570682368 bytes 2026-02-20T21:48:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 616281088 bytes 2026-02-20T21:48:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 619439104 bytes 2026-02-20T21:48:23.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 629124096 bytes 2026-02-20T21:48:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 660690944 bytes 2026-02-20T21:48:23.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 677193728 bytes 2026-02-20T21:48:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 689752064 bytes 2026-02-20T21:48:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 751787008 bytes 2026-02-20T21:48:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 784456704 bytes 2026-02-20T21:48:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 795754496 bytes 2026-02-20T21:48:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 841829376 bytes 2026-02-20T21:48:23.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 859316224 bytes 2026-02-20T21:48:23.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 916924416 bytes 2026-02-20T21:48:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 986083328 bytes 2026-02-20T21:48:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 994977792 bytes 2026-02-20T21:48:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1017627648 bytes 2026-02-20T21:48:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1029010432 bytes 2026-02-20T21:48:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1102317568 bytes 2026-02-20T21:48:24.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1172532224 bytes 2026-02-20T21:48:24.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1194447872 bytes 2026-02-20T21:48:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1216302080 bytes 2026-02-20T21:48:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1239323648 bytes 2026-02-20T21:48:24.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1282363392 bytes 2026-02-20T21:48:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1351810048 bytes 2026-02-20T21:48:24.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1384392704 bytes 2026-02-20T21:48:24.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1410176000 bytes 2026-02-20T21:48:24.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1443639296 bytes 2026-02-20T21:48:24.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1444275200 bytes 2026-02-20T21:48:24.384 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1460303872 bytes 2026-02-20T21:48:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1520062464 bytes 2026-02-20T21:48:24.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1559064576 bytes 2026-02-20T21:48:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1559662592 bytes 2026-02-20T21:48:24.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1632722944 bytes 2026-02-20T21:48:24.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1655536640 bytes 2026-02-20T21:48:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1733984256 bytes 2026-02-20T21:48:24.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1787376640 bytes 2026-02-20T21:48:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1830669312 bytes 2026-02-20T21:48:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 1984721920 bytes 2026-02-20T21:48:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2052201472 bytes 2026-02-20T21:48:24.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2101541888 bytes 2026-02-20T21:48:24.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2198878208 bytes 2026-02-20T21:48:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2275051520 bytes 2026-02-20T21:48:25.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2323079168 bytes 2026-02-20T21:48:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2341653504 bytes 2026-02-20T21:48:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2439640064 bytes 2026-02-20T21:48:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2469463040 bytes 2026-02-20T21:48:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2546115584 bytes 2026-02-20T21:48:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2594045952 bytes 2026-02-20T21:48:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2709877760 bytes 2026-02-20T21:48:25.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2730546176 bytes 2026-02-20T21:48:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2750413824 bytes 2026-02-20T21:48:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2778642432 bytes 2026-02-20T21:48:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2829896704 bytes 2026-02-20T21:48:25.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2840930304 bytes 2026-02-20T21:48:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2932594688 bytes 2026-02-20T21:48:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3015527424 bytes 2026-02-20T21:48:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3040850944 bytes 2026-02-20T21:48:25.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3103571968 bytes 2026-02-20T21:48:25.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3144371200 bytes 2026-02-20T21:48:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3191813120 bytes 2026-02-20T21:48:25.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3222302720 bytes 2026-02-20T21:48:25.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3237910528 bytes 2026-02-20T21:48:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3269143552 bytes 2026-02-20T21:48:25.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3290750976 bytes 2026-02-20T21:48:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3337699328 bytes 2026-02-20T21:48:26.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3400807424 bytes 2026-02-20T21:48:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3441434624 bytes 2026-02-20T21:48:26.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3529310208 bytes 2026-02-20T21:48:26.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3578010624 bytes 2026-02-20T21:48:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3614332928 bytes 2026-02-20T21:48:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3629841408 bytes 2026-02-20T21:48:26.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3673878528 bytes 2026-02-20T21:48:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3674490880 bytes 2026-02-20T21:48:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3712598016 bytes 2026-02-20T21:48:26.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3759688704 bytes 2026-02-20T21:48:26.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3802724352 bytes 2026-02-20T21:48:26.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3881444352 bytes 2026-02-20T21:48:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3949828096 bytes 2026-02-20T21:48:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3958810624 bytes 2026-02-20T21:48:26.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4044374016 bytes 2026-02-20T21:48:26.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4070497280 bytes 2026-02-20T21:48:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4090782720 bytes 2026-02-20T21:48:26.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4187033600 bytes 2026-02-20T21:48:26.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4187643904 bytes 2026-02-20T21:48:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4215061504 bytes 2026-02-20T21:48:26.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4259627008 bytes 2026-02-20T21:48:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4268338176 bytes 2026-02-20T21:48:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4287410176 bytes 2026-02-20T21:48:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4318355456 bytes 2026-02-20T21:48:27.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4336723968 bytes 2026-02-20T21:48:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4361361408 bytes 2026-02-20T21:48:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4395513856 bytes 2026-02-20T21:48:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4420847616 bytes 2026-02-20T21:48:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4451268608 bytes 2026-02-20T21:48:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4463953920 bytes 2026-02-20T21:48:27.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4497321984 bytes 2026-02-20T21:48:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4530093056 bytes 2026-02-20T21:48:27.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4596726784 bytes 2026-02-20T21:48:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4607819776 bytes 2026-02-20T21:48:27.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4622407680 bytes 2026-02-20T21:48:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4642027520 bytes 2026-02-20T21:48:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4710099968 bytes 2026-02-20T21:48:27.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4748447744 bytes 2026-02-20T21:48:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4765185024 bytes 2026-02-20T21:48:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4870020096 bytes 2026-02-20T21:48:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4899598336 bytes 2026-02-20T21:48:27.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4953511936 bytes 2026-02-20T21:48:27.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4971500544 bytes 2026-02-20T21:48:27.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5001874432 bytes 2026-02-20T21:48:27.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5034337280 bytes 2026-02-20T21:48:27.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5051365376 bytes 2026-02-20T21:48:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5076687872 bytes 2026-02-20T21:48:27.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5089945600 bytes 2026-02-20T21:48:28.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5129013248 bytes 2026-02-20T21:48:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5177712640 bytes 2026-02-20T21:48:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5194441728 bytes 2026-02-20T21:48:28.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5250262016 bytes 2026-02-20T21:48:28.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5270984704 bytes 2026-02-20T21:48:28.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5271597056 bytes 2026-02-20T21:48:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5321673728 bytes 2026-02-20T21:48:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5356091392 bytes 2026-02-20T21:48:28.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5383288832 bytes 2026-02-20T21:48:28.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5399745536 bytes 2026-02-20T21:48:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5436710912 bytes 2026-02-20T21:48:28.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5464601600 bytes 2026-02-20T21:48:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5500572672 bytes 2026-02-20T21:48:28.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5506916352 bytes 2026-02-20T21:48:28.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5519767552 bytes 2026-02-20T21:48:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5561815040 bytes 2026-02-20T21:48:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5614263296 bytes 2026-02-20T21:48:28.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5621863424 bytes 2026-02-20T21:48:28.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5650647040 bytes 2026-02-20T21:48:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5690236928 bytes 2026-02-20T21:48:28.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5733804032 bytes 2026-02-20T21:48:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5837705216 bytes 2026-02-20T21:48:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5907889152 bytes 2026-02-20T21:48:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5949406208 bytes 2026-02-20T21:48:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 5990116352 bytes 2026-02-20T21:48:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6085893120 bytes 2026-02-20T21:48:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6184984576 bytes 2026-02-20T21:48:29.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6235844608 bytes 2026-02-20T21:48:29.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6248585216 bytes 2026-02-20T21:48:29.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6262446080 bytes 2026-02-20T21:48:29.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6293428224 bytes 2026-02-20T21:48:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6302659584 bytes 2026-02-20T21:48:29.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6380452864 bytes 2026-02-20T21:48:29.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6411003904 bytes 2026-02-20T21:48:29.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6430218240 bytes 2026-02-20T21:48:29.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6460506112 bytes 2026-02-20T21:48:29.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6504862720 bytes 2026-02-20T21:48:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6581032960 bytes 2026-02-20T21:48:29.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6608128000 bytes 2026-02-20T21:48:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6670324736 bytes 2026-02-20T21:48:29.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6675412992 bytes 2026-02-20T21:48:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6707675136 bytes 2026-02-20T21:48:29.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6727808000 bytes 2026-02-20T21:48:29.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6805637120 bytes 2026-02-20T21:48:29.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6856769536 bytes 2026-02-20T21:48:29.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6886126592 bytes 2026-02-20T21:48:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 6909065216 bytes 2026-02-20T21:48:30.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 6962114560 bytes 2026-02-20T21:48:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 6980645888 bytes 2026-02-20T21:48:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7028923392 bytes 2026-02-20T21:48:30.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7031873536 bytes 2026-02-20T21:48:30.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7037783040 bytes 2026-02-20T21:48:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7051273216 bytes 2026-02-20T21:48:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7051890688 bytes 2026-02-20T21:48:30.274 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7113865216 bytes 2026-02-20T21:48:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7189369856 bytes 2026-02-20T21:48:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7209636864 bytes 2026-02-20T21:48:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7228075008 bytes 2026-02-20T21:48:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7261078528 bytes 2026-02-20T21:48:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7284899840 bytes 2026-02-20T21:48:30.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7350619136 bytes 2026-02-20T21:48:30.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7414364160 bytes 2026-02-20T21:48:30.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7452572672 bytes 2026-02-20T21:48:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7463268352 bytes 2026-02-20T21:48:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7529389056 bytes 2026-02-20T21:48:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7576665088 bytes 2026-02-20T21:48:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7658621952 bytes 2026-02-20T21:48:30.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 7679753216 bytes 2026-02-20T21:48:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 7729496064 bytes 2026-02-20T21:48:30.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 7754903552 bytes 2026-02-20T21:48:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 7755513856 bytes 2026-02-20T21:48:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 7814943744 bytes 2026-02-20T21:48:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 7858521088 bytes 2026-02-20T21:48:31.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 7859133440 bytes 2026-02-20T21:48:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 7933512704 bytes 2026-02-20T21:48:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8058734592 bytes 2026-02-20T21:48:31.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8182425600 bytes 2026-02-20T21:48:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8246219776 bytes 2026-02-20T21:48:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8284898304 bytes 2026-02-20T21:48:31.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8325453824 bytes 2026-02-20T21:48:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:48:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:48:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:48:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_4 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 snapshot 128M 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:48:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_4 128M 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_4 2026-02-20T21:48:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:48:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:48:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_4 snapshot 2026-02-20T21:48:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:48:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2026-02-20T21:48:32.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:48:32.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_4': 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 12976ca82088 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.12976ca82088 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:48:32.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:31 2026 2026-02-20T21:48:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:31 2026 2026-02-20T21:48:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:31 2026 2026-02-20T21:48:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:48:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:48:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 60748c73-296b-4b03-b97e-bb38a2fa0c0a 2026-02-20T21:48:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:48:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:48:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-20T21:48:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:48:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:48:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=5 2026-02-20T21:48:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:48:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:48:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 45845504 bytes 2026-02-20T21:48:32.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 141071360 bytes 2026-02-20T21:48:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 166275072 bytes 2026-02-20T21:48:32.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 185896960 bytes 2026-02-20T21:48:32.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 244965376 bytes 2026-02-20T21:48:32.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 268431360 bytes 2026-02-20T21:48:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 353696768 bytes 2026-02-20T21:48:32.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 407393280 bytes 2026-02-20T21:48:32.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 437649408 bytes 2026-02-20T21:48:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 485505024 bytes 2026-02-20T21:48:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 521009152 bytes 2026-02-20T21:48:33.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 566749184 bytes 2026-02-20T21:48:33.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 576780288 bytes 2026-02-20T21:48:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 631502848 bytes 2026-02-20T21:48:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 648580096 bytes 2026-02-20T21:48:33.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 706351104 bytes 2026-02-20T21:48:33.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 729720832 bytes 2026-02-20T21:48:33.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 798674944 bytes 2026-02-20T21:48:33.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 814951424 bytes 2026-02-20T21:48:33.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 861527040 bytes 2026-02-20T21:48:33.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 923176960 bytes 2026-02-20T21:48:33.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 933478400 bytes 2026-02-20T21:48:33.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 970999808 bytes 2026-02-20T21:48:33.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 987032576 bytes 2026-02-20T21:48:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1015822336 bytes 2026-02-20T21:48:33.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1054270464 bytes 2026-02-20T21:48:33.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1100126208 bytes 2026-02-20T21:48:33.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1137256448 bytes 2026-02-20T21:48:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1212593152 bytes 2026-02-20T21:48:33.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1218690048 bytes 2026-02-20T21:48:33.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1219315712 bytes 2026-02-20T21:48:33.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1244335104 bytes 2026-02-20T21:48:33.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1297985536 bytes 2026-02-20T21:48:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1373446144 bytes 2026-02-20T21:48:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1433569280 bytes 2026-02-20T21:48:34.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1486548992 bytes 2026-02-20T21:48:34.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1500241920 bytes 2026-02-20T21:48:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1582178304 bytes 2026-02-20T21:48:34.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1582803968 bytes 2026-02-20T21:48:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1596444672 bytes 2026-02-20T21:48:34.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1639699456 bytes 2026-02-20T21:48:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1647207424 bytes 2026-02-20T21:48:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1704356864 bytes 2026-02-20T21:48:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1739876352 bytes 2026-02-20T21:48:34.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1803708416 bytes 2026-02-20T21:48:34.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1824207872 bytes 2026-02-20T21:48:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1849272320 bytes 2026-02-20T21:48:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1907938304 bytes 2026-02-20T21:48:34.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2046285824 bytes 2026-02-20T21:48:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2085530624 bytes 2026-02-20T21:48:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2166583296 bytes 2026-02-20T21:48:34.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2225146880 bytes 2026-02-20T21:48:34.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2225760256 bytes 2026-02-20T21:48:34.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2262891520 bytes 2026-02-20T21:48:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2321833984 bytes 2026-02-20T21:48:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2339997696 bytes 2026-02-20T21:48:34.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2388111360 bytes 2026-02-20T21:48:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2433178624 bytes 2026-02-20T21:48:35.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2481496064 bytes 2026-02-20T21:48:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2524039168 bytes 2026-02-20T21:48:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2547721216 bytes 2026-02-20T21:48:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2585447424 bytes 2026-02-20T21:48:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2650684416 bytes 2026-02-20T21:48:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2680649728 bytes 2026-02-20T21:48:35.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2740061184 bytes 2026-02-20T21:48:35.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2813379584 bytes 2026-02-20T21:48:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2832074752 bytes 2026-02-20T21:48:35.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2851930112 bytes 2026-02-20T21:48:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2908371968 bytes 2026-02-20T21:48:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2975004672 bytes 2026-02-20T21:48:35.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2996556800 bytes 2026-02-20T21:48:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3051632640 bytes 2026-02-20T21:48:35.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3115844608 bytes 2026-02-20T21:48:35.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3137206272 bytes 2026-02-20T21:48:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3167775744 bytes 2026-02-20T21:48:35.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3229235200 bytes 2026-02-20T21:48:35.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3282260992 bytes 2026-02-20T21:48:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3302537216 bytes 2026-02-20T21:48:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3395595264 bytes 2026-02-20T21:48:35.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3428619264 bytes 2026-02-20T21:48:36.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3515101184 bytes 2026-02-20T21:48:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3557330944 bytes 2026-02-20T21:48:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3600939008 bytes 2026-02-20T21:48:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3627416576 bytes 2026-02-20T21:48:36.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3647256576 bytes 2026-02-20T21:48:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3762100224 bytes 2026-02-20T21:48:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3782045696 bytes 2026-02-20T21:48:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3843124224 bytes 2026-02-20T21:48:36.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3890051072 bytes 2026-02-20T21:48:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3918085120 bytes 2026-02-20T21:48:36.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4018987008 bytes 2026-02-20T21:48:36.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4048873472 bytes 2026-02-20T21:48:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4074445824 bytes 2026-02-20T21:48:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4097064960 bytes 2026-02-20T21:48:36.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4111541248 bytes 2026-02-20T21:48:36.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4117131264 bytes 2026-02-20T21:48:36.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4183584768 bytes 2026-02-20T21:48:36.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4245738496 bytes 2026-02-20T21:48:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4269803520 bytes 2026-02-20T21:48:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4286896128 bytes 2026-02-20T21:48:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4300694528 bytes 2026-02-20T21:48:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4343260160 bytes 2026-02-20T21:48:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4404622336 bytes 2026-02-20T21:48:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4422792192 bytes 2026-02-20T21:48:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4442960896 bytes 2026-02-20T21:48:37.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4487776256 bytes 2026-02-20T21:48:37.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4516165632 bytes 2026-02-20T21:48:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4536720384 bytes 2026-02-20T21:48:37.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4593528832 bytes 2026-02-20T21:48:37.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4660085760 bytes 2026-02-20T21:48:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4701647872 bytes 2026-02-20T21:48:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4749175808 bytes 2026-02-20T21:48:37.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4816414720 bytes 2026-02-20T21:48:37.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4840062976 bytes 2026-02-20T21:48:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4888337408 bytes 2026-02-20T21:48:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:48:37.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_5 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 snapshot 128M 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_5 128M 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:48:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:48:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_5 2026-02-20T21:48:37.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:48:37.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:37.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:48:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_5 snapshot 2026-02-20T21:48:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:48:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2026-02-20T21:48:38.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:48:38.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-02-20T21:48:38.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_5': 2026-02-20T21:48:38.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 12a3d10c5318 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.12a3d10c5318 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:37 2026 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:37 2026 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:37 2026 2026-02-20T21:48:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 03eab890-30a7-45f9-952e-050bfe363306 2026-02-20T21:48:38.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:48:38.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:48:38.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-20T21:48:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:48:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:48:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=11 2026-02-20T21:48:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:48:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:48:38.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 56908800 bytes 2026-02-20T21:48:38.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 90636288 bytes 2026-02-20T21:48:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 125159424 bytes 2026-02-20T21:48:38.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 157169664 bytes 2026-02-20T21:48:38.729 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8103540441973539) > weight (0.25) 2026-02-20T21:48:38.730 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-20T21:48:38.730 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-20T21:48:38.730 INFO:tasks.rbd_mirror_thrash:waiting for 9.228395262596157 secs before reviving daemons 2026-02-20T21:48:38.731 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T21:48:38.726+0000 7effafb07640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 27022) UID: 1000 2026-02-20T21:48:38.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 179339264 bytes 2026-02-20T21:48:38.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 245958656 bytes 2026-02-20T21:48:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 248853504 bytes 2026-02-20T21:48:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 281354240 bytes 2026-02-20T21:48:38.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 288638976 bytes 2026-02-20T21:48:38.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 321021952 bytes 2026-02-20T21:48:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 396143616 bytes 2026-02-20T21:48:39.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 402291712 bytes 2026-02-20T21:48:39.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 448769024 bytes 2026-02-20T21:48:39.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 466827264 bytes 2026-02-20T21:48:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 514136064 bytes 2026-02-20T21:48:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 520000512 bytes 2026-02-20T21:48:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 527405056 bytes 2026-02-20T21:48:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 552807424 bytes 2026-02-20T21:48:39.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 605700096 bytes 2026-02-20T21:48:39.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 661981184 bytes 2026-02-20T21:48:39.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 689360896 bytes 2026-02-20T21:48:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 740869120 bytes 2026-02-20T21:48:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 836956160 bytes 2026-02-20T21:48:39.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 869882880 bytes 2026-02-20T21:48:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 903265280 bytes 2026-02-20T21:48:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 928633856 bytes 2026-02-20T21:48:39.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1003157504 bytes 2026-02-20T21:48:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1036527616 bytes 2026-02-20T21:48:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1071847424 bytes 2026-02-20T21:48:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1131154432 bytes 2026-02-20T21:48:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1188045824 bytes 2026-02-20T21:48:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1206645760 bytes 2026-02-20T21:48:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1260387328 bytes 2026-02-20T21:48:40.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1269939200 bytes 2026-02-20T21:48:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1320659968 bytes 2026-02-20T21:48:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1328368640 bytes 2026-02-20T21:48:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1328971776 bytes 2026-02-20T21:48:40.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1368752128 bytes 2026-02-20T21:48:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1389882368 bytes 2026-02-20T21:48:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1397356544 bytes 2026-02-20T21:48:40.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1439369216 bytes 2026-02-20T21:48:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1518988288 bytes 2026-02-20T21:48:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1546066944 bytes 2026-02-20T21:48:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1610241024 bytes 2026-02-20T21:48:40.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1620210688 bytes 2026-02-20T21:48:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1703438336 bytes 2026-02-20T21:48:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1734392832 bytes 2026-02-20T21:48:40.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1793815552 bytes 2026-02-20T21:48:40.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1817058304 bytes 2026-02-20T21:48:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1859128320 bytes 2026-02-20T21:48:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 1904933888 bytes 2026-02-20T21:48:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 1936268288 bytes 2026-02-20T21:48:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 1967725568 bytes 2026-02-20T21:48:40.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2027894784 bytes 2026-02-20T21:48:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2116823040 bytes 2026-02-20T21:48:41.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2135015424 bytes 2026-02-20T21:48:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2151341056 bytes 2026-02-20T21:48:41.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2208218112 bytes 2026-02-20T21:48:41.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2283735040 bytes 2026-02-20T21:48:41.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2343811072 bytes 2026-02-20T21:48:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2406351872 bytes 2026-02-20T21:48:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2441477120 bytes 2026-02-20T21:48:41.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2456668160 bytes 2026-02-20T21:48:41.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2505230336 bytes 2026-02-20T21:48:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2597351424 bytes 2026-02-20T21:48:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2639883264 bytes 2026-02-20T21:48:41.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2700335104 bytes 2026-02-20T21:48:41.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2753106944 bytes 2026-02-20T21:48:41.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2771437568 bytes 2026-02-20T21:48:41.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2785606656 bytes 2026-02-20T21:48:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2791003136 bytes 2026-02-20T21:48:41.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2850206720 bytes 2026-02-20T21:48:41.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2896912384 bytes 2026-02-20T21:48:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2937048064 bytes 2026-02-20T21:48:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3018563584 bytes 2026-02-20T21:48:42.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3069114368 bytes 2026-02-20T21:48:42.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3119429632 bytes 2026-02-20T21:48:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3154442240 bytes 2026-02-20T21:48:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3194191872 bytes 2026-02-20T21:48:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3194781696 bytes 2026-02-20T21:48:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3246862336 bytes 2026-02-20T21:48:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3286376448 bytes 2026-02-20T21:48:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3334900736 bytes 2026-02-20T21:48:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3404643328 bytes 2026-02-20T21:48:42.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3411222528 bytes 2026-02-20T21:48:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3486470144 bytes 2026-02-20T21:48:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3525117952 bytes 2026-02-20T21:48:42.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3571792896 bytes 2026-02-20T21:48:42.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3596703744 bytes 2026-02-20T21:48:42.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3598170112 bytes 2026-02-20T21:48:42.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3700857856 bytes 2026-02-20T21:48:42.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3735820288 bytes 2026-02-20T21:48:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3736617984 bytes 2026-02-20T21:48:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3752599552 bytes 2026-02-20T21:48:42.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3766768640 bytes 2026-02-20T21:48:42.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3772078080 bytes 2026-02-20T21:48:43.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3845749760 bytes 2026-02-20T21:48:43.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3957913600 bytes 2026-02-20T21:48:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3994506240 bytes 2026-02-20T21:48:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4085801984 bytes 2026-02-20T21:48:43.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4161228800 bytes 2026-02-20T21:48:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4174247936 bytes 2026-02-20T21:48:43.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4239539200 bytes 2026-02-20T21:48:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4297592832 bytes 2026-02-20T21:48:43.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4382626816 bytes 2026-02-20T21:48:43.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4411685888 bytes 2026-02-20T21:48:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4495248384 bytes 2026-02-20T21:48:43.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4573969408 bytes 2026-02-20T21:48:43.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4610010112 bytes 2026-02-20T21:48:43.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4639652864 bytes 2026-02-20T21:48:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4677210112 bytes 2026-02-20T21:48:43.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4707588096 bytes 2026-02-20T21:48:44.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4773880832 bytes 2026-02-20T21:48:44.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4809044992 bytes 2026-02-20T21:48:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4815343616 bytes 2026-02-20T21:48:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4847003648 bytes 2026-02-20T21:48:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4914715648 bytes 2026-02-20T21:48:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4929238016 bytes 2026-02-20T21:48:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4933388288 bytes 2026-02-20T21:48:44.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4960429056 bytes 2026-02-20T21:48:44.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4985520128 bytes 2026-02-20T21:48:44.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4986132480 bytes 2026-02-20T21:48:44.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5010777088 bytes 2026-02-20T21:48:44.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5041836032 bytes 2026-02-20T21:48:44.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5059814400 bytes 2026-02-20T21:48:44.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5067157504 bytes 2026-02-20T21:48:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5132040192 bytes 2026-02-20T21:48:44.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5175597056 bytes 2026-02-20T21:48:44.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5192566784 bytes 2026-02-20T21:48:44.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5213644800 bytes 2026-02-20T21:48:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5290297344 bytes 2026-02-20T21:48:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5336641536 bytes 2026-02-20T21:48:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5356938240 bytes 2026-02-20T21:48:44.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5411880960 bytes 2026-02-20T21:48:45.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5439781888 bytes 2026-02-20T21:48:45.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5520644096 bytes 2026-02-20T21:48:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5593036800 bytes 2026-02-20T21:48:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5664670720 bytes 2026-02-20T21:48:45.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5680246784 bytes 2026-02-20T21:48:45.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5745463296 bytes 2026-02-20T21:48:45.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5754418176 bytes 2026-02-20T21:48:45.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5805714432 bytes 2026-02-20T21:48:45.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5874020352 bytes 2026-02-20T21:48:45.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5942323200 bytes 2026-02-20T21:48:45.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5954216960 bytes 2026-02-20T21:48:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5977325568 bytes 2026-02-20T21:48:45.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6026371072 bytes 2026-02-20T21:48:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6073138176 bytes 2026-02-20T21:48:45.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6143644672 bytes 2026-02-20T21:48:45.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6206340096 bytes 2026-02-20T21:48:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6234182656 bytes 2026-02-20T21:48:45.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6288673792 bytes 2026-02-20T21:48:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6316190720 bytes 2026-02-20T21:48:45.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6331654144 bytes 2026-02-20T21:48:46.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6414873600 bytes 2026-02-20T21:48:46.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6467268608 bytes 2026-02-20T21:48:46.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6467876864 bytes 2026-02-20T21:48:46.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6474789888 bytes 2026-02-20T21:48:46.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6553908224 bytes 2026-02-20T21:48:46.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6564656128 bytes 2026-02-20T21:48:46.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6603741184 bytes 2026-02-20T21:48:46.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6682887168 bytes 2026-02-20T21:48:46.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6740475904 bytes 2026-02-20T21:48:46.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6825838592 bytes 2026-02-20T21:48:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6848297984 bytes 2026-02-20T21:48:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6880095232 bytes 2026-02-20T21:48:46.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6918743040 bytes 2026-02-20T21:48:46.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6947779584 bytes 2026-02-20T21:48:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6983603200 bytes 2026-02-20T21:48:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7050047488 bytes 2026-02-20T21:48:46.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7070564352 bytes 2026-02-20T21:48:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7081469952 bytes 2026-02-20T21:48:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7154876416 bytes 2026-02-20T21:48:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7221496832 bytes 2026-02-20T21:48:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7267903488 bytes 2026-02-20T21:48:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7302884352 bytes 2026-02-20T21:48:47.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7325263872 bytes 2026-02-20T21:48:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7360934912 bytes 2026-02-20T21:48:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7361536000 bytes 2026-02-20T21:48:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7405500416 bytes 2026-02-20T21:48:47.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7417350144 bytes 2026-02-20T21:48:47.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7478669312 bytes 2026-02-20T21:48:47.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7490920448 bytes 2026-02-20T21:48:47.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7534477312 bytes 2026-02-20T21:48:47.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7557049344 bytes 2026-02-20T21:48:47.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7612981248 bytes 2026-02-20T21:48:47.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7663754240 bytes 2026-02-20T21:48:47.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7725597696 bytes 2026-02-20T21:48:47.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7792240640 bytes 2026-02-20T21:48:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7829912576 bytes 2026-02-20T21:48:47.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7846400000 bytes 2026-02-20T21:48:47.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7915762688 bytes 2026-02-20T21:48:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7964657664 bytes 2026-02-20T21:48:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 7987816448 bytes 2026-02-20T21:48:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8071174144 bytes 2026-02-20T21:48:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8153306112 bytes 2026-02-20T21:48:47.959 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-20T21:48:47.960 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:48:47.960 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-20T21:48:47.960 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T21:48:47.960 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T21:48:47.962 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T21:48:47.963 INFO:tasks.rbd_mirror_thrash:waiting for 90.01943782465526 secs before thrashing 2026-02-20T21:48:47.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8167557120 bytes 2026-02-20T21:48:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8229935104 bytes 2026-02-20T21:48:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8241656832 bytes 2026-02-20T21:48:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8258403328 bytes 2026-02-20T21:48:48.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8273680384 bytes 2026-02-20T21:48:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8295944192 bytes 2026-02-20T21:48:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8327828480 bytes 2026-02-20T21:48:48.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8364489728 bytes 2026-02-20T21:48:48.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8396911616 bytes 2026-02-20T21:48:48.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8474332160 bytes 2026-02-20T21:48:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8513000448 bytes 2026-02-20T21:48:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8603982848 bytes 2026-02-20T21:48:48.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8627243008 bytes 2026-02-20T21:48:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8643880960 bytes 2026-02-20T21:48:48.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8663726080 bytes 2026-02-20T21:48:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8707436544 bytes 2026-02-20T21:48:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8755459072 bytes 2026-02-20T21:48:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8838086656 bytes 2026-02-20T21:48:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8869005312 bytes 2026-02-20T21:48:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 8924764160 bytes 2026-02-20T21:48:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 8962018304 bytes 2026-02-20T21:48:48.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9024186368 bytes 2026-02-20T21:48:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9068182528 bytes 2026-02-20T21:48:49.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9134510080 bytes 2026-02-20T21:48:49.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9206913024 bytes 2026-02-20T21:48:49.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9247931392 bytes 2026-02-20T21:48:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9271226368 bytes 2026-02-20T21:48:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9322071040 bytes 2026-02-20T21:48:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9414754304 bytes 2026-02-20T21:48:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9466954752 bytes 2026-02-20T21:48:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9531820032 bytes 2026-02-20T21:48:49.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9593952256 bytes 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_6 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 snapshot 128M 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_6 128M 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:48:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_6 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_6 snapshot 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:48:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2026-02-20T21:48:49.779 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8538047638943604) > weight (0.25) 2026-02-20T21:48:49.779 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.988368699241068) > weight (0.25) 2026-02-20T21:48:49.779 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4378926514634892) > weight (0.25) 2026-02-20T21:48:49.779 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4522142017476827) > weight (0.25) 2026-02-20T21:48:49.779 INFO:tasks.rbd_mirror_thrash:waiting for 97.08319194548281 secs before thrashing 2026-02-20T21:48:50.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:48:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-02-20T21:48:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_6': 2026-02-20T21:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 12b437d14fb8 2026-02-20T21:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.12b437d14fb8 2026-02-20T21:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:48:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:48:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:49 2026 2026-02-20T21:48:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:49 2026 2026-02-20T21:48:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:49 2026 2026-02-20T21:48:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:48:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:48:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 73eed487-81fa-4ccf-b6ef-2a93a56ed579 2026-02-20T21:48:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:48:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:48:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-20T21:48:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:48:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:48:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:48:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:48:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=14 2026-02-20T21:48:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:48:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:48:50.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 46073856 bytes 2026-02-20T21:48:50.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 60953600 bytes 2026-02-20T21:48:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 121439232 bytes 2026-02-20T21:48:50.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 165047296 bytes 2026-02-20T21:48:50.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 184229888 bytes 2026-02-20T21:48:50.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 228280320 bytes 2026-02-20T21:48:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 277895168 bytes 2026-02-20T21:48:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 327627776 bytes 2026-02-20T21:48:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 348239872 bytes 2026-02-20T21:48:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 365933568 bytes 2026-02-20T21:48:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 388904960 bytes 2026-02-20T21:48:51.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 461725696 bytes 2026-02-20T21:48:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 526973952 bytes 2026-02-20T21:48:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 537568256 bytes 2026-02-20T21:48:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 557877248 bytes 2026-02-20T21:48:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 604315648 bytes 2026-02-20T21:48:51.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 630258688 bytes 2026-02-20T21:48:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 674281472 bytes 2026-02-20T21:48:51.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 719785984 bytes 2026-02-20T21:48:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 735259648 bytes 2026-02-20T21:48:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 745508864 bytes 2026-02-20T21:48:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 802877440 bytes 2026-02-20T21:48:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 841951232 bytes 2026-02-20T21:48:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 857111552 bytes 2026-02-20T21:48:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 914362368 bytes 2026-02-20T21:48:51.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 932872192 bytes 2026-02-20T21:48:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1070922752 bytes 2026-02-20T21:48:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1080581120 bytes 2026-02-20T21:48:51.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1143587840 bytes 2026-02-20T21:48:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1233481728 bytes 2026-02-20T21:48:52.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1265804288 bytes 2026-02-20T21:48:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1339276288 bytes 2026-02-20T21:48:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1416616960 bytes 2026-02-20T21:48:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1435067392 bytes 2026-02-20T21:48:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1470899200 bytes 2026-02-20T21:48:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1519190016 bytes 2026-02-20T21:48:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1639553024 bytes 2026-02-20T21:48:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1699624960 bytes 2026-02-20T21:48:52.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1721058304 bytes 2026-02-20T21:48:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1743980544 bytes 2026-02-20T21:48:52.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1782686720 bytes 2026-02-20T21:48:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1844988928 bytes 2026-02-20T21:48:52.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1919873024 bytes 2026-02-20T21:48:52.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1984035840 bytes 2026-02-20T21:48:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2083676160 bytes 2026-02-20T21:48:52.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2103883776 bytes 2026-02-20T21:48:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2168343552 bytes 2026-02-20T21:48:52.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2216112128 bytes 2026-02-20T21:48:52.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2259719168 bytes 2026-02-20T21:48:53.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2274414592 bytes 2026-02-20T21:48:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2293903360 bytes 2026-02-20T21:48:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2388428800 bytes 2026-02-20T21:48:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2428078080 bytes 2026-02-20T21:48:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2463943680 bytes 2026-02-20T21:48:53.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2547024896 bytes 2026-02-20T21:48:53.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2627784704 bytes 2026-02-20T21:48:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2680496128 bytes 2026-02-20T21:48:53.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2720914432 bytes 2026-02-20T21:48:53.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2781187072 bytes 2026-02-20T21:48:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2791825408 bytes 2026-02-20T21:48:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2846956544 bytes 2026-02-20T21:48:53.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2898673664 bytes 2026-02-20T21:48:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2960877568 bytes 2026-02-20T21:48:53.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3052958720 bytes 2026-02-20T21:48:53.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3093257216 bytes 2026-02-20T21:48:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3155443712 bytes 2026-02-20T21:48:53.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3218536448 bytes 2026-02-20T21:48:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3257540608 bytes 2026-02-20T21:48:54.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3283700736 bytes 2026-02-20T21:48:54.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3300838400 bytes 2026-02-20T21:48:54.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3368776704 bytes 2026-02-20T21:48:54.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3372694528 bytes 2026-02-20T21:48:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3424859136 bytes 2026-02-20T21:48:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3462150144 bytes 2026-02-20T21:48:54.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3490862080 bytes 2026-02-20T21:48:54.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3541411840 bytes 2026-02-20T21:48:54.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3578627072 bytes 2026-02-20T21:48:54.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3620142080 bytes 2026-02-20T21:48:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3644032000 bytes 2026-02-20T21:48:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3652566016 bytes 2026-02-20T21:48:54.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3710667776 bytes 2026-02-20T21:48:54.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3739400192 bytes 2026-02-20T21:48:54.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3814354944 bytes 2026-02-20T21:48:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3835678720 bytes 2026-02-20T21:48:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3895717888 bytes 2026-02-20T21:48:54.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3922951168 bytes 2026-02-20T21:48:54.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3954710528 bytes 2026-02-20T21:48:54.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3989143552 bytes 2026-02-20T21:48:54.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3989765120 bytes 2026-02-20T21:48:54.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4056653824 bytes 2026-02-20T21:48:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4145893376 bytes 2026-02-20T21:48:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4224380928 bytes 2026-02-20T21:48:55.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4275955712 bytes 2026-02-20T21:48:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4367350784 bytes 2026-02-20T21:48:55.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4415750144 bytes 2026-02-20T21:48:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4416377856 bytes 2026-02-20T21:48:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4458230784 bytes 2026-02-20T21:48:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4490112000 bytes 2026-02-20T21:48:55.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4516852736 bytes 2026-02-20T21:48:55.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4625935360 bytes 2026-02-20T21:48:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4672319488 bytes 2026-02-20T21:48:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4747877376 bytes 2026-02-20T21:48:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4768822272 bytes 2026-02-20T21:48:56.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4836189184 bytes 2026-02-20T21:48:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4860076032 bytes 2026-02-20T21:48:56.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4901006336 bytes 2026-02-20T21:48:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4922394624 bytes 2026-02-20T21:48:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4967417856 bytes 2026-02-20T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 5017998336 bytes 2026-02-20T21:48:56.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5054062592 bytes 2026-02-20T21:48:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5119780864 bytes 2026-02-20T21:48:56.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5147556864 bytes 2026-02-20T21:48:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5190284288 bytes 2026-02-20T21:48:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5265797120 bytes 2026-02-20T21:48:56.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5325995008 bytes 2026-02-20T21:48:56.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5443391488 bytes 2026-02-20T21:48:56.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5478752256 bytes 2026-02-20T21:48:56.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5519097856 bytes 2026-02-20T21:48:56.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5528699904 bytes 2026-02-20T21:48:56.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5557224448 bytes 2026-02-20T21:48:56.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5557850112 bytes 2026-02-20T21:48:57.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5564597248 bytes 2026-02-20T21:48:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5599077376 bytes 2026-02-20T21:48:57.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5665134592 bytes 2026-02-20T21:48:57.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5728868352 bytes 2026-02-20T21:48:57.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5812691968 bytes 2026-02-20T21:48:57.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5823876096 bytes 2026-02-20T21:48:57.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5848308736 bytes 2026-02-20T21:48:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5915735040 bytes 2026-02-20T21:48:57.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5986296832 bytes 2026-02-20T21:48:57.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 6068551680 bytes 2026-02-20T21:48:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 6095264768 bytes 2026-02-20T21:48:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 6173968384 bytes 2026-02-20T21:48:57.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 6214559744 bytes 2026-02-20T21:48:57.629 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:48:57.622+0000 7fa5e2658640 -1 reset not still connected to 0x55b90790d6c0 2026-02-20T21:48:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 6242929664 bytes 2026-02-20T21:48:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 6280737792 bytes 2026-02-20T21:48:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 6350998528 bytes 2026-02-20T21:48:57.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6354137088 bytes 2026-02-20T21:48:57.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6369127424 bytes 2026-02-20T21:48:57.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6470845440 bytes 2026-02-20T21:48:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6491476992 bytes 2026-02-20T21:48:58.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6533197824 bytes 2026-02-20T21:48:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6655112192 bytes 2026-02-20T21:48:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6713441280 bytes 2026-02-20T21:48:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6752314368 bytes 2026-02-20T21:48:58.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6840240128 bytes 2026-02-20T21:48:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6854097920 bytes 2026-02-20T21:48:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6879057920 bytes 2026-02-20T21:48:58.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6888048640 bytes 2026-02-20T21:48:58.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6980225024 bytes 2026-02-20T21:48:58.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 7043837952 bytes 2026-02-20T21:48:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 7135788032 bytes 2026-02-20T21:48:58.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 7183465472 bytes 2026-02-20T21:48:58.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 7197178880 bytes 2026-02-20T21:48:58.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 7234755584 bytes 2026-02-20T21:48:58.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 7298146304 bytes 2026-02-20T21:48:58.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 7343942656 bytes 2026-02-20T21:48:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 7416818688 bytes 2026-02-20T21:48:58.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 7491185664 bytes 2026-02-20T21:48:58.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 7589521408 bytes 2026-02-20T21:48:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7663705088 bytes 2026-02-20T21:48:59.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7665923072 bytes 2026-02-20T21:48:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7679557632 bytes 2026-02-20T21:48:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7717306368 bytes 2026-02-20T21:48:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7767910400 bytes 2026-02-20T21:48:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7812818944 bytes 2026-02-20T21:48:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7841538048 bytes 2026-02-20T21:48:59.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7873163264 bytes 2026-02-20T21:48:59.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7945666560 bytes 2026-02-20T21:48:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 8008718336 bytes 2026-02-20T21:48:59.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 8034960384 bytes 2026-02-20T21:48:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 8120736768 bytes 2026-02-20T21:48:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 8166124544 bytes 2026-02-20T21:48:59.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 8203461632 bytes 2026-02-20T21:48:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 8234692608 bytes 2026-02-20T21:48:59.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 8265638912 bytes 2026-02-20T21:48:59.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 8313042944 bytes 2026-02-20T21:48:59.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 8320919552 bytes 2026-02-20T21:48:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 8344164352 bytes 2026-02-20T21:48:59.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 8382000128 bytes 2026-02-20T21:48:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 8443794432 bytes 2026-02-20T21:48:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 8444434432 bytes 2026-02-20T21:49:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 8471882752 bytes 2026-02-20T21:49:00.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 8530229248 bytes 2026-02-20T21:49:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 8546652160 bytes 2026-02-20T21:49:00.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 8572436480 bytes 2026-02-20T21:49:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8597532672 bytes 2026-02-20T21:49:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8652277760 bytes 2026-02-20T21:49:00.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8671501312 bytes 2026-02-20T21:49:00.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8727779328 bytes 2026-02-20T21:49:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8762596352 bytes 2026-02-20T21:49:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8798180352 bytes 2026-02-20T21:49:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8812837888 bytes 2026-02-20T21:49:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8878497792 bytes 2026-02-20T21:49:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8921120768 bytes 2026-02-20T21:49:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8983343104 bytes 2026-02-20T21:49:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 9018667008 bytes 2026-02-20T21:49:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 9019293696 bytes 2026-02-20T21:49:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 9086148608 bytes 2026-02-20T21:49:00.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 9124613120 bytes 2026-02-20T21:49:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 9184553984 bytes 2026-02-20T21:49:00.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 9235252224 bytes 2026-02-20T21:49:00.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 9235889152 bytes 2026-02-20T21:49:01.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 9306430464 bytes 2026-02-20T21:49:01.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 9350387712 bytes 2026-02-20T21:49:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 9365003264 bytes 2026-02-20T21:49:01.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 9415490560 bytes 2026-02-20T21:49:01.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 9420092416 bytes 2026-02-20T21:49:01.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 9476639744 bytes 2026-02-20T21:49:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9530068992 bytes 2026-02-20T21:49:01.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9561543680 bytes 2026-02-20T21:49:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9592591360 bytes 2026-02-20T21:49:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9626329088 bytes 2026-02-20T21:49:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9675616256 bytes 2026-02-20T21:49:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9727461376 bytes 2026-02-20T21:49:01.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9753681920 bytes 2026-02-20T21:49:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9792451584 bytes 2026-02-20T21:49:01.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9812323328 bytes 2026-02-20T21:49:01.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9865607168 bytes 2026-02-20T21:49:01.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9869004800 bytes 2026-02-20T21:49:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9930450944 bytes 2026-02-20T21:49:01.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9959552000 bytes 2026-02-20T21:49:01.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9996086272 bytes 2026-02-20T21:49:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 10055912448 bytes 2026-02-20T21:49:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 10060156928 bytes 2026-02-20T21:49:02.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 10090326016 bytes 2026-02-20T21:49:02.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 10163731456 bytes 2026-02-20T21:49:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 10206088192 bytes 2026-02-20T21:49:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 10253168640 bytes 2026-02-20T21:49:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 10290954240 bytes 2026-02-20T21:49:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 10326991872 bytes 2026-02-20T21:49:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 10409389056 bytes 2026-02-20T21:49:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 10423540736 bytes 2026-02-20T21:49:02.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 10461821952 bytes 2026-02-20T21:49:02.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 10513955840 bytes 2026-02-20T21:49:02.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 10613259264 bytes 2026-02-20T21:49:02.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 10665226240 bytes 2026-02-20T21:49:02.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 10681008128 bytes 2026-02-20T21:49:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 10724871168 bytes 2026-02-20T21:49:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 10755489792 bytes 2026-02-20T21:49:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 10874608640 bytes 2026-02-20T21:49:02.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10875226112 bytes 2026-02-20T21:49:02.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 11002522624 bytes 2026-02-20T21:49:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 11103588352 bytes 2026-02-20T21:49:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 11206814720 bytes 2026-02-20T21:49:03.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 11251379200 bytes 2026-02-20T21:49:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 11307225088 bytes 2026-02-20T21:49:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 11341440000 bytes 2026-02-20T21:49:03.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 11360992256 bytes 2026-02-20T21:49:03.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 11477616640 bytes 2026-02-20T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25100 IOs, 11501371392 bytes 2026-02-20T21:49:03.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25200 IOs, 11552003072 bytes 2026-02-20T21:49:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25300 IOs, 11628408832 bytes 2026-02-20T21:49:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25400 IOs, 11656088576 bytes 2026-02-20T21:49:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25500 IOs, 11664668672 bytes 2026-02-20T21:49:03.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25600 IOs, 11686919168 bytes 2026-02-20T21:49:03.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25700 IOs, 11689385984 bytes 2026-02-20T21:49:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25800 IOs, 11769853952 bytes 2026-02-20T21:49:03.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25900 IOs, 11796234240 bytes 2026-02-20T21:49:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26000 IOs, 11840977920 bytes 2026-02-20T21:49:03.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26100 IOs, 11895782400 bytes 2026-02-20T21:49:03.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26200 IOs, 11930260480 bytes 2026-02-20T21:49:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26300 IOs, 11993099264 bytes 2026-02-20T21:49:03.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26400 IOs, 11994844160 bytes 2026-02-20T21:49:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26500 IOs, 12049175552 bytes 2026-02-20T21:49:04.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26600 IOs, 12100459520 bytes 2026-02-20T21:49:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26700 IOs, 12150211584 bytes 2026-02-20T21:49:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26800 IOs, 12192183296 bytes 2026-02-20T21:49:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26900 IOs, 12218962944 bytes 2026-02-20T21:49:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27000 IOs, 12258469888 bytes 2026-02-20T21:49:04.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27100 IOs, 12280168448 bytes 2026-02-20T21:49:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27200 IOs, 12299648000 bytes 2026-02-20T21:49:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27300 IOs, 12317854720 bytes 2026-02-20T21:49:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27400 IOs, 12337790976 bytes 2026-02-20T21:49:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27500 IOs, 12374508544 bytes 2026-02-20T21:49:04.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_7 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 snapshot 128M 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_7 128M 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:04.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_7 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_7 snapshot 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2026-02-20T21:49:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:49:05.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-02-20T21:49:05.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_7': 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 12c0e4d2377a 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.12c0e4d2377a 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:04 2026 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:04 2026 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:04 2026 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: de4e320f-58cf-4d0e-b463-cea2ccd8e6c3 2026-02-20T21:49:05.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:49:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:49:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-20T21:49:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:05.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:49:05.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:49:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T21:49:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:49:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:49:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 28932096 bytes 2026-02-20T21:49:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 57225216 bytes 2026-02-20T21:49:05.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 114030592 bytes 2026-02-20T21:49:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 163297280 bytes 2026-02-20T21:49:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 206595072 bytes 2026-02-20T21:49:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 283984896 bytes 2026-02-20T21:49:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 352009216 bytes 2026-02-20T21:49:05.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 439221248 bytes 2026-02-20T21:49:06.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 533001216 bytes 2026-02-20T21:49:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 561915904 bytes 2026-02-20T21:49:06.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 596311040 bytes 2026-02-20T21:49:06.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 636326912 bytes 2026-02-20T21:49:06.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 657502208 bytes 2026-02-20T21:49:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 698229760 bytes 2026-02-20T21:49:06.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 708492288 bytes 2026-02-20T21:49:06.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 819230720 bytes 2026-02-20T21:49:06.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 860948480 bytes 2026-02-20T21:49:06.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 891038720 bytes 2026-02-20T21:49:06.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 963864576 bytes 2026-02-20T21:49:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 999042048 bytes 2026-02-20T21:49:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1016307712 bytes 2026-02-20T21:49:06.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1033402368 bytes 2026-02-20T21:49:06.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1092835328 bytes 2026-02-20T21:49:06.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1114999808 bytes 2026-02-20T21:49:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1131686912 bytes 2026-02-20T21:49:06.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1255860224 bytes 2026-02-20T21:49:06.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1308195840 bytes 2026-02-20T21:49:06.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1349642240 bytes 2026-02-20T21:49:07.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1353662464 bytes 2026-02-20T21:49:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1424922624 bytes 2026-02-20T21:49:07.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1470915584 bytes 2026-02-20T21:49:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1495485440 bytes 2026-02-20T21:49:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1514389504 bytes 2026-02-20T21:49:07.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1556904960 bytes 2026-02-20T21:49:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1628982272 bytes 2026-02-20T21:49:07.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1654035456 bytes 2026-02-20T21:49:07.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1713155072 bytes 2026-02-20T21:49:07.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1713770496 bytes 2026-02-20T21:49:07.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1814753280 bytes 2026-02-20T21:49:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1905630208 bytes 2026-02-20T21:49:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1915497472 bytes 2026-02-20T21:49:07.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1955614720 bytes 2026-02-20T21:49:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1963421696 bytes 2026-02-20T21:49:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2001404928 bytes 2026-02-20T21:49:07.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2044014592 bytes 2026-02-20T21:49:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2102064128 bytes 2026-02-20T21:49:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2174656512 bytes 2026-02-20T21:49:07.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2232080384 bytes 2026-02-20T21:49:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2274183168 bytes 2026-02-20T21:49:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2295928832 bytes 2026-02-20T21:49:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2305006592 bytes 2026-02-20T21:49:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2329208832 bytes 2026-02-20T21:49:08.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2377652224 bytes 2026-02-20T21:49:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2378297344 bytes 2026-02-20T21:49:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2475468800 bytes 2026-02-20T21:49:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2541219840 bytes 2026-02-20T21:49:08.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2556527616 bytes 2026-02-20T21:49:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2628404224 bytes 2026-02-20T21:49:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2640686080 bytes 2026-02-20T21:49:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2694798336 bytes 2026-02-20T21:49:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2738492416 bytes 2026-02-20T21:49:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2800637952 bytes 2026-02-20T21:49:08.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2838184960 bytes 2026-02-20T21:49:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2886034432 bytes 2026-02-20T21:49:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2932544512 bytes 2026-02-20T21:49:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2938471424 bytes 2026-02-20T21:49:08.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2978205696 bytes 2026-02-20T21:49:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3039208448 bytes 2026-02-20T21:49:08.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3065837568 bytes 2026-02-20T21:49:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3139724288 bytes 2026-02-20T21:49:09.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3185320960 bytes 2026-02-20T21:49:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3218708480 bytes 2026-02-20T21:49:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3246043136 bytes 2026-02-20T21:49:09.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3291372544 bytes 2026-02-20T21:49:09.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3328516096 bytes 2026-02-20T21:49:09.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3399757824 bytes 2026-02-20T21:49:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3440039936 bytes 2026-02-20T21:49:09.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3476220928 bytes 2026-02-20T21:49:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3476826112 bytes 2026-02-20T21:49:09.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3517586432 bytes 2026-02-20T21:49:09.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3524403200 bytes 2026-02-20T21:49:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3562187776 bytes 2026-02-20T21:49:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3590694912 bytes 2026-02-20T21:49:09.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3594046464 bytes 2026-02-20T21:49:09.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3681363968 bytes 2026-02-20T21:49:09.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3707843584 bytes 2026-02-20T21:49:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3731304448 bytes 2026-02-20T21:49:09.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3755748352 bytes 2026-02-20T21:49:09.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3789007872 bytes 2026-02-20T21:49:09.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3860788224 bytes 2026-02-20T21:49:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3910010880 bytes 2026-02-20T21:49:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3919518720 bytes 2026-02-20T21:49:10.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3975908352 bytes 2026-02-20T21:49:10.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3996500992 bytes 2026-02-20T21:49:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3997124608 bytes 2026-02-20T21:49:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4071965696 bytes 2026-02-20T21:49:10.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4104525824 bytes 2026-02-20T21:49:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4159766528 bytes 2026-02-20T21:49:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4180986880 bytes 2026-02-20T21:49:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4197653504 bytes 2026-02-20T21:49:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4227119104 bytes 2026-02-20T21:49:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4230964224 bytes 2026-02-20T21:49:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4250141696 bytes 2026-02-20T21:49:10.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4258091008 bytes 2026-02-20T21:49:10.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4292087808 bytes 2026-02-20T21:49:10.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4337059840 bytes 2026-02-20T21:49:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4391109632 bytes 2026-02-20T21:49:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4427233280 bytes 2026-02-20T21:49:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4441093120 bytes 2026-02-20T21:49:10.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4441721856 bytes 2026-02-20T21:49:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4491392000 bytes 2026-02-20T21:49:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4534011904 bytes 2026-02-20T21:49:10.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4571339776 bytes 2026-02-20T21:49:10.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4601016320 bytes 2026-02-20T21:49:11.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4639289344 bytes 2026-02-20T21:49:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4704582656 bytes 2026-02-20T21:49:11.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4727533568 bytes 2026-02-20T21:49:11.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4774708224 bytes 2026-02-20T21:49:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4810572800 bytes 2026-02-20T21:49:11.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4835289088 bytes 2026-02-20T21:49:11.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4898374656 bytes 2026-02-20T21:49:11.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4947039232 bytes 2026-02-20T21:49:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4992959488 bytes 2026-02-20T21:49:11.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4993581056 bytes 2026-02-20T21:49:11.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5030391808 bytes 2026-02-20T21:49:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5030996992 bytes 2026-02-20T21:49:11.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5089690624 bytes 2026-02-20T21:49:11.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5107545088 bytes 2026-02-20T21:49:11.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5233240064 bytes 2026-02-20T21:49:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5290460160 bytes 2026-02-20T21:49:11.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5330633728 bytes 2026-02-20T21:49:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5356081152 bytes 2026-02-20T21:49:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5393354752 bytes 2026-02-20T21:49:11.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5448619008 bytes 2026-02-20T21:49:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5499979776 bytes 2026-02-20T21:49:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5550164992 bytes 2026-02-20T21:49:12.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5620649984 bytes 2026-02-20T21:49:12.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5692310528 bytes 2026-02-20T21:49:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5692940288 bytes 2026-02-20T21:49:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5699041280 bytes 2026-02-20T21:49:12.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5728006144 bytes 2026-02-20T21:49:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5765041152 bytes 2026-02-20T21:49:12.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5821305856 bytes 2026-02-20T21:49:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5842984960 bytes 2026-02-20T21:49:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5906722816 bytes 2026-02-20T21:49:12.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5922918400 bytes 2026-02-20T21:49:12.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5942754304 bytes 2026-02-20T21:49:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5963606016 bytes 2026-02-20T21:49:12.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5987964928 bytes 2026-02-20T21:49:12.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6063885312 bytes 2026-02-20T21:49:12.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6086025216 bytes 2026-02-20T21:49:12.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6120569856 bytes 2026-02-20T21:49:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6162720768 bytes 2026-02-20T21:49:12.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6187512832 bytes 2026-02-20T21:49:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6228739072 bytes 2026-02-20T21:49:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6263525376 bytes 2026-02-20T21:49:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6344791040 bytes 2026-02-20T21:49:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6352189440 bytes 2026-02-20T21:49:13.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6396509184 bytes 2026-02-20T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6442867712 bytes 2026-02-20T21:49:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6500002816 bytes 2026-02-20T21:49:13.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6577657856 bytes 2026-02-20T21:49:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6669144064 bytes 2026-02-20T21:49:13.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6689846272 bytes 2026-02-20T21:49:13.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6727533568 bytes 2026-02-20T21:49:13.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6738624512 bytes 2026-02-20T21:49:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6770192384 bytes 2026-02-20T21:49:13.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6792716288 bytes 2026-02-20T21:49:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6850234368 bytes 2026-02-20T21:49:13.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6920418304 bytes 2026-02-20T21:49:13.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6976066560 bytes 2026-02-20T21:49:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7083413504 bytes 2026-02-20T21:49:13.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7140648960 bytes 2026-02-20T21:49:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7163869184 bytes 2026-02-20T21:49:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7224879104 bytes 2026-02-20T21:49:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7270553600 bytes 2026-02-20T21:49:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7301566464 bytes 2026-02-20T21:49:14.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7380046848 bytes 2026-02-20T21:49:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7426513920 bytes 2026-02-20T21:49:14.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7445066752 bytes 2026-02-20T21:49:14.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7449551872 bytes 2026-02-20T21:49:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7496203264 bytes 2026-02-20T21:49:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7540733952 bytes 2026-02-20T21:49:14.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7563764736 bytes 2026-02-20T21:49:14.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7653639168 bytes 2026-02-20T21:49:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7661911040 bytes 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_8 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 snapshot 128M 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:49:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_8 128M 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_8 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_8 snapshot 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2026-02-20T21:49:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:49:15.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-02-20T21:49:15.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_8': 2026-02-20T21:49:15.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:49:15.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:49:15.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 12cc4117b7a6 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.12cc4117b7a6 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:14 2026 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:14 2026 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:14 2026 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:49:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:49:15.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: b550a116-b65a-457c-854a-75ae61a3215e 2026-02-20T21:49:15.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:49:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:49:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-20T21:49:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:49:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:49:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=14 2026-02-20T21:49:15.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:49:15.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:49:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 24022016 bytes 2026-02-20T21:49:15.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 35538944 bytes 2026-02-20T21:49:15.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 86763520 bytes 2026-02-20T21:49:15.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 139292672 bytes 2026-02-20T21:49:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 169601024 bytes 2026-02-20T21:49:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 222728192 bytes 2026-02-20T21:49:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 259084288 bytes 2026-02-20T21:49:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 294737920 bytes 2026-02-20T21:49:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 390095872 bytes 2026-02-20T21:49:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 497730560 bytes 2026-02-20T21:49:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 510000128 bytes 2026-02-20T21:49:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 549696512 bytes 2026-02-20T21:49:16.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 606094336 bytes 2026-02-20T21:49:16.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 606728192 bytes 2026-02-20T21:49:16.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 774930432 bytes 2026-02-20T21:49:16.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 817709056 bytes 2026-02-20T21:49:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 856517632 bytes 2026-02-20T21:49:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 937444352 bytes 2026-02-20T21:49:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 967430144 bytes 2026-02-20T21:49:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1011643392 bytes 2026-02-20T21:49:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1051327488 bytes 2026-02-20T21:49:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1109866496 bytes 2026-02-20T21:49:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1143567360 bytes 2026-02-20T21:49:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1169894400 bytes 2026-02-20T21:49:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1203675136 bytes 2026-02-20T21:49:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1254680576 bytes 2026-02-20T21:49:16.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1285827584 bytes 2026-02-20T21:49:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1318851584 bytes 2026-02-20T21:49:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1320751104 bytes 2026-02-20T21:49:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1376187392 bytes 2026-02-20T21:49:17.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1404413952 bytes 2026-02-20T21:49:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1456605184 bytes 2026-02-20T21:49:17.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1469297664 bytes 2026-02-20T21:49:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1491330048 bytes 2026-02-20T21:49:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1514925056 bytes 2026-02-20T21:49:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1567795200 bytes 2026-02-20T21:49:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1615559680 bytes 2026-02-20T21:49:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1652339712 bytes 2026-02-20T21:49:17.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1668968448 bytes 2026-02-20T21:49:17.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1719673856 bytes 2026-02-20T21:49:17.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1754510336 bytes 2026-02-20T21:49:17.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1789217792 bytes 2026-02-20T21:49:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1832604672 bytes 2026-02-20T21:49:17.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1904600064 bytes 2026-02-20T21:49:17.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1951380480 bytes 2026-02-20T21:49:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1992002560 bytes 2026-02-20T21:49:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2054571008 bytes 2026-02-20T21:49:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2055221248 bytes 2026-02-20T21:49:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2057059328 bytes 2026-02-20T21:49:17.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2101490688 bytes 2026-02-20T21:49:17.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2141184000 bytes 2026-02-20T21:49:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2148790272 bytes 2026-02-20T21:49:18.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2173512704 bytes 2026-02-20T21:49:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2243374080 bytes 2026-02-20T21:49:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2261415936 bytes 2026-02-20T21:49:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2297099264 bytes 2026-02-20T21:49:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2332819456 bytes 2026-02-20T21:49:18.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2387182592 bytes 2026-02-20T21:49:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2441901056 bytes 2026-02-20T21:49:18.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2468867072 bytes 2026-02-20T21:49:18.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2502157312 bytes 2026-02-20T21:49:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2608245760 bytes 2026-02-20T21:49:18.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2665678848 bytes 2026-02-20T21:49:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2671730688 bytes 2026-02-20T21:49:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2692381696 bytes 2026-02-20T21:49:18.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2734092288 bytes 2026-02-20T21:49:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2793418752 bytes 2026-02-20T21:49:18.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2818156544 bytes 2026-02-20T21:49:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2818779136 bytes 2026-02-20T21:49:18.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2886434816 bytes 2026-02-20T21:49:18.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2927715328 bytes 2026-02-20T21:49:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2943674368 bytes 2026-02-20T21:49:18.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2985352192 bytes 2026-02-20T21:49:18.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3018784768 bytes 2026-02-20T21:49:19.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3100248064 bytes 2026-02-20T21:49:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3145153536 bytes 2026-02-20T21:49:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3155593216 bytes 2026-02-20T21:49:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3193260032 bytes 2026-02-20T21:49:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3247603712 bytes 2026-02-20T21:49:19.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3273346048 bytes 2026-02-20T21:49:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3278143488 bytes 2026-02-20T21:49:19.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3418914816 bytes 2026-02-20T21:49:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3438338048 bytes 2026-02-20T21:49:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3501585408 bytes 2026-02-20T21:49:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3521448960 bytes 2026-02-20T21:49:19.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3537961984 bytes 2026-02-20T21:49:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3554638848 bytes 2026-02-20T21:49:19.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3555274752 bytes 2026-02-20T21:49:19.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3565933568 bytes 2026-02-20T21:49:19.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3593883648 bytes 2026-02-20T21:49:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3621200896 bytes 2026-02-20T21:49:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3666225152 bytes 2026-02-20T21:49:19.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3685013504 bytes 2026-02-20T21:49:19.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3717391360 bytes 2026-02-20T21:49:19.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3753814016 bytes 2026-02-20T21:49:19.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3799813120 bytes 2026-02-20T21:49:19.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3825113088 bytes 2026-02-20T21:49:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3827973120 bytes 2026-02-20T21:49:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3874257920 bytes 2026-02-20T21:49:20.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3935926272 bytes 2026-02-20T21:49:20.100 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4013789184 bytes 2026-02-20T21:49:20.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4046694400 bytes 2026-02-20T21:49:20.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4094964736 bytes 2026-02-20T21:49:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4123790336 bytes 2026-02-20T21:49:20.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4156894208 bytes 2026-02-20T21:49:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4198661120 bytes 2026-02-20T21:49:20.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4232956928 bytes 2026-02-20T21:49:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4290131968 bytes 2026-02-20T21:49:20.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4373376000 bytes 2026-02-20T21:49:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4404576256 bytes 2026-02-20T21:49:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4449261568 bytes 2026-02-20T21:49:20.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4560927744 bytes 2026-02-20T21:49:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4656942080 bytes 2026-02-20T21:49:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4718916608 bytes 2026-02-20T21:49:20.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4751176704 bytes 2026-02-20T21:49:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4803061760 bytes 2026-02-20T21:49:20.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4862710784 bytes 2026-02-20T21:49:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4882618368 bytes 2026-02-20T21:49:20.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4930280448 bytes 2026-02-20T21:49:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4941007872 bytes 2026-02-20T21:49:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4941636608 bytes 2026-02-20T21:49:21.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4987188224 bytes 2026-02-20T21:49:21.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5031745536 bytes 2026-02-20T21:49:21.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5084741632 bytes 2026-02-20T21:49:21.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5138838528 bytes 2026-02-20T21:49:21.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5188674560 bytes 2026-02-20T21:49:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5287729152 bytes 2026-02-20T21:49:21.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5308724224 bytes 2026-02-20T21:49:21.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5354022912 bytes 2026-02-20T21:49:21.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5417426944 bytes 2026-02-20T21:49:21.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5418046464 bytes 2026-02-20T21:49:21.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5485435904 bytes 2026-02-20T21:49:21.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5494154240 bytes 2026-02-20T21:49:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5494745088 bytes 2026-02-20T21:49:21.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5554502656 bytes 2026-02-20T21:49:21.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5568846848 bytes 2026-02-20T21:49:21.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5572177920 bytes 2026-02-20T21:49:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5607496704 bytes 2026-02-20T21:49:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5636707328 bytes 2026-02-20T21:49:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5714890752 bytes 2026-02-20T21:49:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5786638336 bytes 2026-02-20T21:49:22.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5808370688 bytes 2026-02-20T21:49:22.100 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5816463360 bytes 2026-02-20T21:49:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5897066496 bytes 2026-02-20T21:49:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5943309312 bytes 2026-02-20T21:49:22.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5955884032 bytes 2026-02-20T21:49:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6003739648 bytes 2026-02-20T21:49:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6028706816 bytes 2026-02-20T21:49:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6069953536 bytes 2026-02-20T21:49:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6099117056 bytes 2026-02-20T21:49:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6129585152 bytes 2026-02-20T21:49:22.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6158908416 bytes 2026-02-20T21:49:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6177958912 bytes 2026-02-20T21:49:22.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6215794688 bytes 2026-02-20T21:49:22.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6276527104 bytes 2026-02-20T21:49:22.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6297643008 bytes 2026-02-20T21:49:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6370618368 bytes 2026-02-20T21:49:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6381519872 bytes 2026-02-20T21:49:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6430538752 bytes 2026-02-20T21:49:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6470167552 bytes 2026-02-20T21:49:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6498779136 bytes 2026-02-20T21:49:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6587879424 bytes 2026-02-20T21:49:23.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6659766272 bytes 2026-02-20T21:49:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6679979008 bytes 2026-02-20T21:49:23.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6711969792 bytes 2026-02-20T21:49:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6754788352 bytes 2026-02-20T21:49:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6787009536 bytes 2026-02-20T21:49:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6877128704 bytes 2026-02-20T21:49:23.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6946993152 bytes 2026-02-20T21:49:23.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7001472000 bytes 2026-02-20T21:49:23.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7012510720 bytes 2026-02-20T21:49:23.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7047803904 bytes 2026-02-20T21:49:23.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7103552512 bytes 2026-02-20T21:49:23.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7127596032 bytes 2026-02-20T21:49:23.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7139674112 bytes 2026-02-20T21:49:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7186609152 bytes 2026-02-20T21:49:23.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7230225408 bytes 2026-02-20T21:49:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7338587136 bytes 2026-02-20T21:49:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7382432768 bytes 2026-02-20T21:49:23.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7414986752 bytes 2026-02-20T21:49:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7472276480 bytes 2026-02-20T21:49:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7512054784 bytes 2026-02-20T21:49:24.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7541352448 bytes 2026-02-20T21:49:24.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7602600960 bytes 2026-02-20T21:49:24.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7638216704 bytes 2026-02-20T21:49:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7656269824 bytes 2026-02-20T21:49:24.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7674109952 bytes 2026-02-20T21:49:24.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7694959616 bytes 2026-02-20T21:49:24.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7752195072 bytes 2026-02-20T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7774608384 bytes 2026-02-20T21:49:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7818066944 bytes 2026-02-20T21:49:24.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7885999104 bytes 2026-02-20T21:49:24.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7948635136 bytes 2026-02-20T21:49:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8030432256 bytes 2026-02-20T21:49:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8105107456 bytes 2026-02-20T21:49:24.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8151522304 bytes 2026-02-20T21:49:24.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8157745152 bytes 2026-02-20T21:49:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8189928448 bytes 2026-02-20T21:49:24.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8243617792 bytes 2026-02-20T21:49:24.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8244226048 bytes 2026-02-20T21:49:24.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8289458176 bytes 2026-02-20T21:49:24.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8322681856 bytes 2026-02-20T21:49:25.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8361909248 bytes 2026-02-20T21:49:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8368857088 bytes 2026-02-20T21:49:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8445011968 bytes 2026-02-20T21:49:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8505317376 bytes 2026-02-20T21:49:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8574754816 bytes 2026-02-20T21:49:25.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8592680960 bytes 2026-02-20T21:49:25.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8649268224 bytes 2026-02-20T21:49:25.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8673084416 bytes 2026-02-20T21:49:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8697852928 bytes 2026-02-20T21:49:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8721511424 bytes 2026-02-20T21:49:25.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8737896448 bytes 2026-02-20T21:49:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8748411904 bytes 2026-02-20T21:49:25.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8786470912 bytes 2026-02-20T21:49:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 8839744512 bytes 2026-02-20T21:49:25.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 8868276224 bytes 2026-02-20T21:49:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 8901968896 bytes 2026-02-20T21:49:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 8971200512 bytes 2026-02-20T21:49:25.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9056390144 bytes 2026-02-20T21:49:25.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9150969856 bytes 2026-02-20T21:49:25.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9162231808 bytes 2026-02-20T21:49:26.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9211147264 bytes 2026-02-20T21:49:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9276713984 bytes 2026-02-20T21:49:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9306352640 bytes 2026-02-20T21:49:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9395378176 bytes 2026-02-20T21:49:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9457954816 bytes 2026-02-20T21:49:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9477557248 bytes 2026-02-20T21:49:26.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9486842880 bytes 2026-02-20T21:49:26.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9517698048 bytes 2026-02-20T21:49:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9530424320 bytes 2026-02-20T21:49:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9544290304 bytes 2026-02-20T21:49:26.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9628329984 bytes 2026-02-20T21:49:26.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9645285376 bytes 2026-02-20T21:49:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9699101696 bytes 2026-02-20T21:49:26.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 9773510656 bytes 2026-02-20T21:49:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 9825252352 bytes 2026-02-20T21:49:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 9845245952 bytes 2026-02-20T21:49:26.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 9868359680 bytes 2026-02-20T21:49:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 9888690176 bytes 2026-02-20T21:49:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 9940465664 bytes 2026-02-20T21:49:26.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10030556160 bytes 2026-02-20T21:49:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 10052441088 bytes 2026-02-20T21:49:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 10062128128 bytes 2026-02-20T21:49:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10065830912 bytes 2026-02-20T21:49:27.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10121831424 bytes 2026-02-20T21:49:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10134072320 bytes 2026-02-20T21:49:27.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10150975488 bytes 2026-02-20T21:49:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 10235784192 bytes 2026-02-20T21:49:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 10271926272 bytes 2026-02-20T21:49:27.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25100 IOs, 10318248960 bytes 2026-02-20T21:49:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25200 IOs, 10402930688 bytes 2026-02-20T21:49:27.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25300 IOs, 10415797248 bytes 2026-02-20T21:49:27.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25400 IOs, 10515300352 bytes 2026-02-20T21:49:27.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25500 IOs, 10562552832 bytes 2026-02-20T21:49:27.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25600 IOs, 10563167232 bytes 2026-02-20T21:49:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25700 IOs, 10598906880 bytes 2026-02-20T21:49:27.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25800 IOs, 10658022400 bytes 2026-02-20T21:49:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25900 IOs, 10670468096 bytes 2026-02-20T21:49:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26000 IOs, 10728152064 bytes 2026-02-20T21:49:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26100 IOs, 10750170112 bytes 2026-02-20T21:49:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26200 IOs, 10821818368 bytes 2026-02-20T21:49:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26300 IOs, 10885652480 bytes 2026-02-20T21:49:28.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26400 IOs, 10926390272 bytes 2026-02-20T21:49:28.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26500 IOs, 10978859008 bytes 2026-02-20T21:49:28.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26600 IOs, 11008955392 bytes 2026-02-20T21:49:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26700 IOs, 11026971648 bytes 2026-02-20T21:49:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26800 IOs, 11068385280 bytes 2026-02-20T21:49:28.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26900 IOs, 11086692352 bytes 2026-02-20T21:49:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27000 IOs, 11173145600 bytes 2026-02-20T21:49:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27100 IOs, 11245195264 bytes 2026-02-20T21:49:28.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27200 IOs, 11280638976 bytes 2026-02-20T21:49:28.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27300 IOs, 11378659328 bytes 2026-02-20T21:49:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27400 IOs, 11426698240 bytes 2026-02-20T21:49:28.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27500 IOs, 11485612032 bytes 2026-02-20T21:49:28.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27600 IOs, 11486229504 bytes 2026-02-20T21:49:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27700 IOs, 11510799360 bytes 2026-02-20T21:49:28.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27800 IOs, 11530179584 bytes 2026-02-20T21:49:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27900 IOs, 11535303680 bytes 2026-02-20T21:49:28.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28000 IOs, 11598285824 bytes 2026-02-20T21:49:28.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28100 IOs, 11647325184 bytes 2026-02-20T21:49:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28200 IOs, 11677402112 bytes 2026-02-20T21:49:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28300 IOs, 11698055168 bytes 2026-02-20T21:49:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28400 IOs, 11762404352 bytes 2026-02-20T21:49:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28500 IOs, 11780438016 bytes 2026-02-20T21:49:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28600 IOs, 11849803776 bytes 2026-02-20T21:49:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28700 IOs, 11938235392 bytes 2026-02-20T21:49:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28800 IOs, 12019829760 bytes 2026-02-20T21:49:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28900 IOs, 12074781696 bytes 2026-02-20T21:49:29.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29000 IOs, 12130805760 bytes 2026-02-20T21:49:29.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29100 IOs, 12142860288 bytes 2026-02-20T21:49:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29200 IOs, 12143449088 bytes 2026-02-20T21:49:29.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29300 IOs, 12190642176 bytes 2026-02-20T21:49:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:29400 IOs, 12213575680 bytes 2026-02-20T21:49:29.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:49:29.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:49:29.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:49:29.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:29.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:49:29.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_9 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 snapshot 128M 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_9 128M 2026-02-20T21:49:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_9 2026-02-20T21:49:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:49:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:49:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_9 snapshot 2026-02-20T21:49:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:49:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2026-02-20T21:49:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:49:30.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-02-20T21:49:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_9': 2026-02-20T21:49:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:49:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:49:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:49:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 12d8e293f447 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.12d8e293f447 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:29 2026 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:29 2026 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:29 2026 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:49:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 3db04467-75e8-4151-b6e5-29a86d819303 2026-02-20T21:49:30.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:49:30.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:49:30.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-20T21:49:30.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:30.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:30.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:49:30.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:49:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:49:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:49:30.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:49:30.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 39204864 bytes 2026-02-20T21:49:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 72691712 bytes 2026-02-20T21:49:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 113845248 bytes 2026-02-20T21:49:30.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 155653120 bytes 2026-02-20T21:49:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 235053056 bytes 2026-02-20T21:49:30.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 278238208 bytes 2026-02-20T21:49:30.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 307936256 bytes 2026-02-20T21:49:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 311860224 bytes 2026-02-20T21:49:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 368148480 bytes 2026-02-20T21:49:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 396575744 bytes 2026-02-20T21:49:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 430382080 bytes 2026-02-20T21:49:31.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 474152960 bytes 2026-02-20T21:49:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 524778496 bytes 2026-02-20T21:49:31.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 561121280 bytes 2026-02-20T21:49:31.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 582896640 bytes 2026-02-20T21:49:31.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 615922688 bytes 2026-02-20T21:49:31.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 672633856 bytes 2026-02-20T21:49:31.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 749943808 bytes 2026-02-20T21:49:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 804021248 bytes 2026-02-20T21:49:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 840994816 bytes 2026-02-20T21:49:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 897196032 bytes 2026-02-20T21:49:31.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 943661056 bytes 2026-02-20T21:49:31.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1008547840 bytes 2026-02-20T21:49:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1024146432 bytes 2026-02-20T21:49:31.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1073006592 bytes 2026-02-20T21:49:31.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1101169664 bytes 2026-02-20T21:49:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1126593536 bytes 2026-02-20T21:49:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1208759296 bytes 2026-02-20T21:49:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1259116544 bytes 2026-02-20T21:49:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1297257472 bytes 2026-02-20T21:49:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1332105216 bytes 2026-02-20T21:49:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1383378944 bytes 2026-02-20T21:49:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1383991296 bytes 2026-02-20T21:49:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1436541952 bytes 2026-02-20T21:49:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1484880896 bytes 2026-02-20T21:49:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1511812096 bytes 2026-02-20T21:49:32.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1548386304 bytes 2026-02-20T21:49:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1595377664 bytes 2026-02-20T21:49:32.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1689508864 bytes 2026-02-20T21:49:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1718409216 bytes 2026-02-20T21:49:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1794817024 bytes 2026-02-20T21:49:32.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1795433472 bytes 2026-02-20T21:49:32.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1817836544 bytes 2026-02-20T21:49:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1909618688 bytes 2026-02-20T21:49:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2007102464 bytes 2026-02-20T21:49:32.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2106822656 bytes 2026-02-20T21:49:32.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2107425792 bytes 2026-02-20T21:49:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2178528256 bytes 2026-02-20T21:49:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2202478592 bytes 2026-02-20T21:49:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2236378112 bytes 2026-02-20T21:49:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2245009408 bytes 2026-02-20T21:49:33.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2306167808 bytes 2026-02-20T21:49:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2347976704 bytes 2026-02-20T21:49:33.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2369232896 bytes 2026-02-20T21:49:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2429553664 bytes 2026-02-20T21:49:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2517410816 bytes 2026-02-20T21:49:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2518026240 bytes 2026-02-20T21:49:33.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2535726080 bytes 2026-02-20T21:49:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2556416000 bytes 2026-02-20T21:49:33.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2622582784 bytes 2026-02-20T21:49:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2650074112 bytes 2026-02-20T21:49:33.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2665690112 bytes 2026-02-20T21:49:33.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2733956096 bytes 2026-02-20T21:49:33.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2872570880 bytes 2026-02-20T21:49:33.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2902934528 bytes 2026-02-20T21:49:33.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2928353280 bytes 2026-02-20T21:49:33.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2986909696 bytes 2026-02-20T21:49:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3014986752 bytes 2026-02-20T21:49:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3021059072 bytes 2026-02-20T21:49:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3087644672 bytes 2026-02-20T21:49:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3105454080 bytes 2026-02-20T21:49:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3183401984 bytes 2026-02-20T21:49:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3213553664 bytes 2026-02-20T21:49:34.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3301225472 bytes 2026-02-20T21:49:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3356827648 bytes 2026-02-20T21:49:34.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3472679936 bytes 2026-02-20T21:49:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3503323136 bytes 2026-02-20T21:49:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3503949824 bytes 2026-02-20T21:49:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3530872832 bytes 2026-02-20T21:49:34.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3550881792 bytes 2026-02-20T21:49:34.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3591716864 bytes 2026-02-20T21:49:34.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3667999744 bytes 2026-02-20T21:49:34.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3722041344 bytes 2026-02-20T21:49:34.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3745231872 bytes 2026-02-20T21:49:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3752828928 bytes 2026-02-20T21:49:34.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3760771072 bytes 2026-02-20T21:49:34.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3813020672 bytes 2026-02-20T21:49:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3857974272 bytes 2026-02-20T21:49:34.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3885737984 bytes 2026-02-20T21:49:34.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3953227776 bytes 2026-02-20T21:49:35.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4023475200 bytes 2026-02-20T21:49:35.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4048699392 bytes 2026-02-20T21:49:35.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4096058368 bytes 2026-02-20T21:49:35.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4096674816 bytes 2026-02-20T21:49:35.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4188265472 bytes 2026-02-20T21:49:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4201147392 bytes 2026-02-20T21:49:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4238256128 bytes 2026-02-20T21:49:35.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4317828096 bytes 2026-02-20T21:49:35.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4378311680 bytes 2026-02-20T21:49:35.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4378941440 bytes 2026-02-20T21:49:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4413263872 bytes 2026-02-20T21:49:35.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4436467712 bytes 2026-02-20T21:49:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4455904256 bytes 2026-02-20T21:49:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4485367808 bytes 2026-02-20T21:49:35.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4525639680 bytes 2026-02-20T21:49:35.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4572811264 bytes 2026-02-20T21:49:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4613559296 bytes 2026-02-20T21:49:35.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4689437696 bytes 2026-02-20T21:49:35.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4690074624 bytes 2026-02-20T21:49:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4698296320 bytes 2026-02-20T21:49:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4722209792 bytes 2026-02-20T21:49:36.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4769394688 bytes 2026-02-20T21:49:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4832358400 bytes 2026-02-20T21:49:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4896856064 bytes 2026-02-20T21:49:36.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4947212288 bytes 2026-02-20T21:49:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4980047872 bytes 2026-02-20T21:49:36.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5054036992 bytes 2026-02-20T21:49:36.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5080041472 bytes 2026-02-20T21:49:36.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5121431552 bytes 2026-02-20T21:49:36.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5180176384 bytes 2026-02-20T21:49:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5248603136 bytes 2026-02-20T21:49:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:49:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:49:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:49:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_10 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 snapshot 128M 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_10 128M 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:49:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:49:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:49:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:49:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_10 2026-02-20T21:49:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:49:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:49:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_10 snapshot 2026-02-20T21:49:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:49:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:36.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2026-02-20T21:49:36.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:49:36.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-02-20T21:49:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_10': 2026-02-20T21:49:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:49:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:49:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:49:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 12e47c065c6e 2026-02-20T21:49:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.12e47c065c6e 2026-02-20T21:49:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:49:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:49:36.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:49:36.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:49:36.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:36 2026 2026-02-20T21:49:36.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:36 2026 2026-02-20T21:49:36.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:36 2026 2026-02-20T21:49:36.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:49:36.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:49:36.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 73f7f3f5-fe4b-491f-8f59-209e2587d369 2026-02-20T21:49:36.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:49:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:49:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-20T21:49:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:49:36.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:49:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=7 2026-02-20T21:49:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:49:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:49:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 5950464 bytes 2026-02-20T21:49:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 12126208 bytes 2026-02-20T21:49:36.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 82417664 bytes 2026-02-20T21:49:36.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 185663488 bytes 2026-02-20T21:49:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 197830656 bytes 2026-02-20T21:49:37.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 280160256 bytes 2026-02-20T21:49:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 389182464 bytes 2026-02-20T21:49:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 420109312 bytes 2026-02-20T21:49:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 441837568 bytes 2026-02-20T21:49:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 506630144 bytes 2026-02-20T21:49:37.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 564837376 bytes 2026-02-20T21:49:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 603045888 bytes 2026-02-20T21:49:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 629142528 bytes 2026-02-20T21:49:37.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 681835520 bytes 2026-02-20T21:49:37.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 744574976 bytes 2026-02-20T21:49:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 776124416 bytes 2026-02-20T21:49:37.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 799151104 bytes 2026-02-20T21:49:37.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 879261696 bytes 2026-02-20T21:49:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 913089536 bytes 2026-02-20T21:49:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 927865856 bytes 2026-02-20T21:49:37.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 968704000 bytes 2026-02-20T21:49:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1038449664 bytes 2026-02-20T21:49:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1112705024 bytes 2026-02-20T21:49:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1144387584 bytes 2026-02-20T21:49:37.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1194767360 bytes 2026-02-20T21:49:37.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1251879936 bytes 2026-02-20T21:49:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1345157120 bytes 2026-02-20T21:49:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1364781056 bytes 2026-02-20T21:49:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1445690368 bytes 2026-02-20T21:49:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1458614272 bytes 2026-02-20T21:49:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1527132160 bytes 2026-02-20T21:49:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1614056448 bytes 2026-02-20T21:49:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1626971136 bytes 2026-02-20T21:49:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1653920768 bytes 2026-02-20T21:49:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1718121472 bytes 2026-02-20T21:49:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1730483200 bytes 2026-02-20T21:49:38.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1746993152 bytes 2026-02-20T21:49:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1809128448 bytes 2026-02-20T21:49:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1876500480 bytes 2026-02-20T21:49:38.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1898397696 bytes 2026-02-20T21:49:38.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1952620544 bytes 2026-02-20T21:49:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1959090176 bytes 2026-02-20T21:49:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2077908992 bytes 2026-02-20T21:49:38.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2111162368 bytes 2026-02-20T21:49:38.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2142912512 bytes 2026-02-20T21:49:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2179576832 bytes 2026-02-20T21:49:38.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2183920640 bytes 2026-02-20T21:49:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2208449536 bytes 2026-02-20T21:49:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2233483264 bytes 2026-02-20T21:49:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2254436352 bytes 2026-02-20T21:49:39.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2317847552 bytes 2026-02-20T21:49:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2342205440 bytes 2026-02-20T21:49:39.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2374726656 bytes 2026-02-20T21:49:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2421142528 bytes 2026-02-20T21:49:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2445331456 bytes 2026-02-20T21:49:39.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2471091200 bytes 2026-02-20T21:49:39.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2479419392 bytes 2026-02-20T21:49:39.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2502368256 bytes 2026-02-20T21:49:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2545624064 bytes 2026-02-20T21:49:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2590915584 bytes 2026-02-20T21:49:39.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2644920320 bytes 2026-02-20T21:49:39.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2679145472 bytes 2026-02-20T21:49:39.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2748286976 bytes 2026-02-20T21:49:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2748915712 bytes 2026-02-20T21:49:39.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2751307776 bytes 2026-02-20T21:49:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2764571648 bytes 2026-02-20T21:49:39.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2815974400 bytes 2026-02-20T21:49:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2926198784 bytes 2026-02-20T21:49:39.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3002277888 bytes 2026-02-20T21:49:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3002882048 bytes 2026-02-20T21:49:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3062022144 bytes 2026-02-20T21:49:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3108028416 bytes 2026-02-20T21:49:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3131368448 bytes 2026-02-20T21:49:40.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3131984896 bytes 2026-02-20T21:49:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3193001984 bytes 2026-02-20T21:49:40.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3194589184 bytes 2026-02-20T21:49:40.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3270689792 bytes 2026-02-20T21:49:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3290621952 bytes 2026-02-20T21:49:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3362050048 bytes 2026-02-20T21:49:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3408307200 bytes 2026-02-20T21:49:40.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3487291392 bytes 2026-02-20T21:49:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3505043456 bytes 2026-02-20T21:49:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3613160448 bytes 2026-02-20T21:49:40.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3633361920 bytes 2026-02-20T21:49:40.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3658947584 bytes 2026-02-20T21:49:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3683495936 bytes 2026-02-20T21:49:40.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3705502720 bytes 2026-02-20T21:49:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3745206272 bytes 2026-02-20T21:49:40.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3787416576 bytes 2026-02-20T21:49:40.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3847686144 bytes 2026-02-20T21:49:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3868207104 bytes 2026-02-20T21:49:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3900210176 bytes 2026-02-20T21:49:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3911914496 bytes 2026-02-20T21:49:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3933611008 bytes 2026-02-20T21:49:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3981380608 bytes 2026-02-20T21:49:41.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4037831680 bytes 2026-02-20T21:49:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4072854528 bytes 2026-02-20T21:49:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4127678464 bytes 2026-02-20T21:49:41.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4142338048 bytes 2026-02-20T21:49:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4188738560 bytes 2026-02-20T21:49:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4204456960 bytes 2026-02-20T21:49:41.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4247824384 bytes 2026-02-20T21:49:41.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4276736000 bytes 2026-02-20T21:49:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4308852736 bytes 2026-02-20T21:49:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4346935296 bytes 2026-02-20T21:49:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4402692096 bytes 2026-02-20T21:49:41.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4415394816 bytes 2026-02-20T21:49:41.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4456091648 bytes 2026-02-20T21:49:41.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4526957568 bytes 2026-02-20T21:49:41.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4564719616 bytes 2026-02-20T21:49:41.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4599351296 bytes 2026-02-20T21:49:41.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4620513280 bytes 2026-02-20T21:49:41.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4664588288 bytes 2026-02-20T21:49:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4718412800 bytes 2026-02-20T21:49:41.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4754909184 bytes 2026-02-20T21:49:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4786797568 bytes 2026-02-20T21:49:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4838098944 bytes 2026-02-20T21:49:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4858508288 bytes 2026-02-20T21:49:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4862073856 bytes 2026-02-20T21:49:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4947893248 bytes 2026-02-20T21:49:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4986217472 bytes 2026-02-20T21:49:42.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5016474624 bytes 2026-02-20T21:49:42.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5045618688 bytes 2026-02-20T21:49:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5135554560 bytes 2026-02-20T21:49:42.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5159192576 bytes 2026-02-20T21:49:42.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5178222592 bytes 2026-02-20T21:49:42.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5216336896 bytes 2026-02-20T21:49:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5258927104 bytes 2026-02-20T21:49:42.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5281282048 bytes 2026-02-20T21:49:42.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5339503616 bytes 2026-02-20T21:49:42.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5386803200 bytes 2026-02-20T21:49:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5482405888 bytes 2026-02-20T21:49:43.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5538499584 bytes 2026-02-20T21:49:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5547834368 bytes 2026-02-20T21:49:43.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5592147968 bytes 2026-02-20T21:49:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5609690112 bytes 2026-02-20T21:49:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5665546240 bytes 2026-02-20T21:49:43.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5677891584 bytes 2026-02-20T21:49:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5758069760 bytes 2026-02-20T21:49:43.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5826723840 bytes 2026-02-20T21:49:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5863837696 bytes 2026-02-20T21:49:43.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5872659456 bytes 2026-02-20T21:49:43.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5873288192 bytes 2026-02-20T21:49:43.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5873916928 bytes 2026-02-20T21:49:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5964276736 bytes 2026-02-20T21:49:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6007966720 bytes 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_11 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 snapshot 128M 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_11 128M 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_11 2026-02-20T21:49:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:49:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:49:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_11 snapshot 2026-02-20T21:49:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:49:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_11 snapshot 2026-02-20T21:49:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:49:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_11 2026-02-20T21:49:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_11': 2026-02-20T21:49:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:49:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 12f093191611 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.12f093191611 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:43 2026 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:43 2026 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:43 2026 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:49:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:49:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: bb0f4c60-0c8c-41c3-9fc2-067109d74988 2026-02-20T21:49:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:49:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:49:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_11 2026-02-20T21:49:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:44.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:49:44.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:49:44.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=5 2026-02-20T21:49:44.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:49:44.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:49:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 77916160 bytes 2026-02-20T21:49:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 128525312 bytes 2026-02-20T21:49:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 213857280 bytes 2026-02-20T21:49:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 267384832 bytes 2026-02-20T21:49:45.023 INFO:tasks.ceph.cluster2.osd.1.trial107.stderr:2026-02-20T21:49:45.017+0000 7f6b9c59b640 -1 reset not still connected to 0x564e812ce340 2026-02-20T21:49:45.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 311306240 bytes 2026-02-20T21:49:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 440300544 bytes 2026-02-20T21:49:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 494702592 bytes 2026-02-20T21:49:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 583853056 bytes 2026-02-20T21:49:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 597848064 bytes 2026-02-20T21:49:45.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 638974976 bytes 2026-02-20T21:49:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 688815104 bytes 2026-02-20T21:49:45.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 739118080 bytes 2026-02-20T21:49:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 790383616 bytes 2026-02-20T21:49:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 881963008 bytes 2026-02-20T21:49:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 951350272 bytes 2026-02-20T21:49:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 977734656 bytes 2026-02-20T21:49:45.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 1021005824 bytes 2026-02-20T21:49:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 1087917056 bytes 2026-02-20T21:49:45.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 1107753984 bytes 2026-02-20T21:49:46.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1146479616 bytes 2026-02-20T21:49:46.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1173090304 bytes 2026-02-20T21:49:46.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1251837952 bytes 2026-02-20T21:49:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1340748800 bytes 2026-02-20T21:49:46.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1346613248 bytes 2026-02-20T21:49:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1374476288 bytes 2026-02-20T21:49:46.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1423857664 bytes 2026-02-20T21:49:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1503621120 bytes 2026-02-20T21:49:46.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1578341376 bytes 2026-02-20T21:49:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1615811584 bytes 2026-02-20T21:49:46.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1702560768 bytes 2026-02-20T21:49:46.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1748932608 bytes 2026-02-20T21:49:46.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1810534400 bytes 2026-02-20T21:49:46.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1834294272 bytes 2026-02-20T21:49:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1885031424 bytes 2026-02-20T21:49:46.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1901568000 bytes 2026-02-20T21:49:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1915327488 bytes 2026-02-20T21:49:46.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1978346496 bytes 2026-02-20T21:49:46.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1994251264 bytes 2026-02-20T21:49:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 2019631104 bytes 2026-02-20T21:49:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 2082403328 bytes 2026-02-20T21:49:47.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 2115678208 bytes 2026-02-20T21:49:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2165322752 bytes 2026-02-20T21:49:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2168921088 bytes 2026-02-20T21:49:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2224655360 bytes 2026-02-20T21:49:47.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2234484736 bytes 2026-02-20T21:49:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2310753280 bytes 2026-02-20T21:49:47.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2325763072 bytes 2026-02-20T21:49:47.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2411149312 bytes 2026-02-20T21:49:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2426723328 bytes 2026-02-20T21:49:47.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2449359872 bytes 2026-02-20T21:49:47.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2534050816 bytes 2026-02-20T21:49:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2629742592 bytes 2026-02-20T21:49:47.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2692926464 bytes 2026-02-20T21:49:47.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2717470720 bytes 2026-02-20T21:49:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2737223680 bytes 2026-02-20T21:49:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2806535168 bytes 2026-02-20T21:49:47.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2849719296 bytes 2026-02-20T21:49:47.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2908168192 bytes 2026-02-20T21:49:47.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 3005595648 bytes 2026-02-20T21:49:48.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 3051948032 bytes 2026-02-20T21:49:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 3112940544 bytes 2026-02-20T21:49:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 3201746944 bytes 2026-02-20T21:49:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 3275674624 bytes 2026-02-20T21:49:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3276302336 bytes 2026-02-20T21:49:48.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3313112064 bytes 2026-02-20T21:49:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3350580224 bytes 2026-02-20T21:49:48.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3407497216 bytes 2026-02-20T21:49:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3429720064 bytes 2026-02-20T21:49:48.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3500773376 bytes 2026-02-20T21:49:48.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3527032832 bytes 2026-02-20T21:49:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3527659520 bytes 2026-02-20T21:49:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3621762048 bytes 2026-02-20T21:49:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3656554496 bytes 2026-02-20T21:49:48.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3704596480 bytes 2026-02-20T21:49:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3747774464 bytes 2026-02-20T21:49:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3766500352 bytes 2026-02-20T21:49:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3816744960 bytes 2026-02-20T21:49:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3832019968 bytes 2026-02-20T21:49:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3880478720 bytes 2026-02-20T21:49:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3881096192 bytes 2026-02-20T21:49:48.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3898606592 bytes 2026-02-20T21:49:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3962947584 bytes 2026-02-20T21:49:49.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3989372928 bytes 2026-02-20T21:49:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 4047306752 bytes 2026-02-20T21:49:49.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 4071549952 bytes 2026-02-20T21:49:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 4079684608 bytes 2026-02-20T21:49:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 4136484864 bytes 2026-02-20T21:49:49.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4186532864 bytes 2026-02-20T21:49:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4228021248 bytes 2026-02-20T21:49:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4286487552 bytes 2026-02-20T21:49:49.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4332457984 bytes 2026-02-20T21:49:49.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4393126912 bytes 2026-02-20T21:49:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4414060544 bytes 2026-02-20T21:49:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4455814144 bytes 2026-02-20T21:49:49.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_12 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 snapshot 128M 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:49.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_12 128M 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:49:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:49:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:49:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:49:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_12 2026-02-20T21:49:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:49:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:49:49.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_12 snapshot 2026-02-20T21:49:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:49:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:49.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_12 snapshot 2026-02-20T21:49:50.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:49:50.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_12 2026-02-20T21:49:50.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_12': 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 12fcb587e073 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.12fcb587e073 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:49 2026 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:49 2026 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:49 2026 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: ffaa636b-9c5c-4539-965b-b5870a492843 2026-02-20T21:49:50.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:49:50.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:49:50.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_12 2026-02-20T21:49:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:49:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:49:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=8 2026-02-20T21:49:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:49:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:49:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 14337024 bytes 2026-02-20T21:49:50.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 56302592 bytes 2026-02-20T21:49:50.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 56898560 bytes 2026-02-20T21:49:50.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 122758144 bytes 2026-02-20T21:49:50.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 130624512 bytes 2026-02-20T21:49:50.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 145355776 bytes 2026-02-20T21:49:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 155606016 bytes 2026-02-20T21:49:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 189693952 bytes 2026-02-20T21:49:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 255431680 bytes 2026-02-20T21:49:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 285924352 bytes 2026-02-20T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 327048192 bytes 2026-02-20T21:49:51.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 392560640 bytes 2026-02-20T21:49:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 467691520 bytes 2026-02-20T21:49:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 514571264 bytes 2026-02-20T21:49:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 563528704 bytes 2026-02-20T21:49:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 631668736 bytes 2026-02-20T21:49:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 647364608 bytes 2026-02-20T21:49:51.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 657177600 bytes 2026-02-20T21:49:51.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 669534208 bytes 2026-02-20T21:49:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 683122688 bytes 2026-02-20T21:49:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 740170752 bytes 2026-02-20T21:49:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 774061056 bytes 2026-02-20T21:49:51.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 877081600 bytes 2026-02-20T21:49:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 964897792 bytes 2026-02-20T21:49:51.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 974912512 bytes 2026-02-20T21:49:51.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1057509376 bytes 2026-02-20T21:49:51.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1079296000 bytes 2026-02-20T21:49:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1109508096 bytes 2026-02-20T21:49:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1131652096 bytes 2026-02-20T21:49:52.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1177076736 bytes 2026-02-20T21:49:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1220723712 bytes 2026-02-20T21:49:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1279722496 bytes 2026-02-20T21:49:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1359529984 bytes 2026-02-20T21:49:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1384312832 bytes 2026-02-20T21:49:52.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1417933824 bytes 2026-02-20T21:49:52.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1512200192 bytes 2026-02-20T21:49:52.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1595527168 bytes 2026-02-20T21:49:52.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1630401536 bytes 2026-02-20T21:49:52.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1689354240 bytes 2026-02-20T21:49:52.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1739851776 bytes 2026-02-20T21:49:52.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1788238848 bytes 2026-02-20T21:49:52.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1826208768 bytes 2026-02-20T21:49:52.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1841799168 bytes 2026-02-20T21:49:52.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1887050752 bytes 2026-02-20T21:49:52.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1906680832 bytes 2026-02-20T21:49:52.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1924339712 bytes 2026-02-20T21:49:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1962672128 bytes 2026-02-20T21:49:52.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1986915328 bytes 2026-02-20T21:49:52.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2013650944 bytes 2026-02-20T21:49:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2037792768 bytes 2026-02-20T21:49:53.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2054600704 bytes 2026-02-20T21:49:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2093319168 bytes 2026-02-20T21:49:53.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2117518336 bytes 2026-02-20T21:49:53.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2152135680 bytes 2026-02-20T21:49:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2193088512 bytes 2026-02-20T21:49:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2211066880 bytes 2026-02-20T21:49:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2240729088 bytes 2026-02-20T21:49:53.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2302528512 bytes 2026-02-20T21:49:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2319679488 bytes 2026-02-20T21:49:53.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2342049792 bytes 2026-02-20T21:49:53.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2380779520 bytes 2026-02-20T21:49:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2445263872 bytes 2026-02-20T21:49:53.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2535824384 bytes 2026-02-20T21:49:53.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2595397632 bytes 2026-02-20T21:49:53.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2623873024 bytes 2026-02-20T21:49:53.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2679914496 bytes 2026-02-20T21:49:53.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2754108416 bytes 2026-02-20T21:49:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2801537024 bytes 2026-02-20T21:49:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2851048448 bytes 2026-02-20T21:49:53.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2919523328 bytes 2026-02-20T21:49:53.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2973863936 bytes 2026-02-20T21:49:54.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2995771392 bytes 2026-02-20T21:49:54.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3036895232 bytes 2026-02-20T21:49:54.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3058704384 bytes 2026-02-20T21:49:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3107934208 bytes 2026-02-20T21:49:54.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3148681216 bytes 2026-02-20T21:49:54.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3149298688 bytes 2026-02-20T21:49:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3204950016 bytes 2026-02-20T21:49:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3287095296 bytes 2026-02-20T21:49:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3389228032 bytes 2026-02-20T21:49:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3433597952 bytes 2026-02-20T21:49:54.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3454343168 bytes 2026-02-20T21:49:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3492218880 bytes 2026-02-20T21:49:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3514842112 bytes 2026-02-20T21:49:54.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3546535936 bytes 2026-02-20T21:49:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3599671296 bytes 2026-02-20T21:49:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3651576832 bytes 2026-02-20T21:49:54.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3694011392 bytes 2026-02-20T21:49:54.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3721396224 bytes 2026-02-20T21:49:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3773516800 bytes 2026-02-20T21:49:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3835763712 bytes 2026-02-20T21:49:54.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3881673728 bytes 2026-02-20T21:49:54.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3992582144 bytes 2026-02-20T21:49:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4015286272 bytes 2026-02-20T21:49:55.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4028826624 bytes 2026-02-20T21:49:55.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4093012992 bytes 2026-02-20T21:49:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4118172672 bytes 2026-02-20T21:49:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4181657600 bytes 2026-02-20T21:49:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4211879936 bytes 2026-02-20T21:49:55.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4286821376 bytes 2026-02-20T21:49:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4341252096 bytes 2026-02-20T21:49:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4386590720 bytes 2026-02-20T21:49:55.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4392646656 bytes 2026-02-20T21:49:55.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4430157824 bytes 2026-02-20T21:49:55.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4468744192 bytes 2026-02-20T21:49:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4565650432 bytes 2026-02-20T21:49:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4584013824 bytes 2026-02-20T21:49:55.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4600304640 bytes 2026-02-20T21:49:55.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4644722688 bytes 2026-02-20T21:49:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4737546240 bytes 2026-02-20T21:49:55.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4768141312 bytes 2026-02-20T21:49:55.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4805609472 bytes 2026-02-20T21:49:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4904725504 bytes 2026-02-20T21:49:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4944431104 bytes 2026-02-20T21:49:55.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4966337536 bytes 2026-02-20T21:49:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4982998016 bytes 2026-02-20T21:49:56.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4983612416 bytes 2026-02-20T21:49:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5007898624 bytes 2026-02-20T21:49:56.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5045091328 bytes 2026-02-20T21:49:56.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5087194112 bytes 2026-02-20T21:49:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5124975616 bytes 2026-02-20T21:49:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5154366464 bytes 2026-02-20T21:49:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5200116736 bytes 2026-02-20T21:49:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5252810752 bytes 2026-02-20T21:49:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5279694848 bytes 2026-02-20T21:49:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5377362944 bytes 2026-02-20T21:49:56.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5468865536 bytes 2026-02-20T21:49:56.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5507461120 bytes 2026-02-20T21:49:56.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5567196160 bytes 2026-02-20T21:49:56.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5593375744 bytes 2026-02-20T21:49:56.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5608799232 bytes 2026-02-20T21:49:56.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5648869376 bytes 2026-02-20T21:49:57.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5697715200 bytes 2026-02-20T21:49:57.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5745465344 bytes 2026-02-20T21:49:57.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5782547456 bytes 2026-02-20T21:49:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5846302720 bytes 2026-02-20T21:49:57.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5879531520 bytes 2026-02-20T21:49:57.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5895149568 bytes 2026-02-20T21:49:57.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5980560384 bytes 2026-02-20T21:49:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6009053184 bytes 2026-02-20T21:49:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6117263360 bytes 2026-02-20T21:49:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6164007936 bytes 2026-02-20T21:49:57.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6242692096 bytes 2026-02-20T21:49:57.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6260442112 bytes 2026-02-20T21:49:57.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6297651200 bytes 2026-02-20T21:49:57.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6359090176 bytes 2026-02-20T21:49:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6404550656 bytes 2026-02-20T21:49:57.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6479439872 bytes 2026-02-20T21:49:57.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6538748928 bytes 2026-02-20T21:49:57.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6611367936 bytes 2026-02-20T21:49:57.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6667443200 bytes 2026-02-20T21:49:57.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6671867904 bytes 2026-02-20T21:49:58.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6730055680 bytes 2026-02-20T21:49:58.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6834738176 bytes 2026-02-20T21:49:58.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6920048640 bytes 2026-02-20T21:49:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6977813504 bytes 2026-02-20T21:49:58.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 7030936576 bytes 2026-02-20T21:49:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 7059725312 bytes 2026-02-20T21:49:58.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 7087554560 bytes 2026-02-20T21:49:58.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 7124911104 bytes 2026-02-20T21:49:58.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7157223424 bytes 2026-02-20T21:49:58.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7225082880 bytes 2026-02-20T21:49:58.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7250527232 bytes 2026-02-20T21:49:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7272906752 bytes 2026-02-20T21:49:58.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7308528640 bytes 2026-02-20T21:49:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7317784576 bytes 2026-02-20T21:49:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7350959104 bytes 2026-02-20T21:49:58.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7382528000 bytes 2026-02-20T21:49:58.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:49:58.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_13 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 snapshot 128M 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:49:58.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_13 128M 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:58.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:49:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:49:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:49:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:49:58.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_13 2026-02-20T21:49:58.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:49:58.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:58.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:49:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:49:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_13 snapshot 2026-02-20T21:49:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:49:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:49:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_13 snapshot 2026-02-20T21:49:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:49:59.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_13 2026-02-20T21:49:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_13': 2026-02-20T21:49:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:49:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:49:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:49:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 1308734cbdab 2026-02-20T21:49:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.1308734cbdab 2026-02-20T21:49:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:49:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:49:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:49:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:49:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:58 2026 2026-02-20T21:49:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:58 2026 2026-02-20T21:49:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:58 2026 2026-02-20T21:49:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:49:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:49:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 9cfb07c7-2733-4bb0-9788-9c939376ecdc 2026-02-20T21:49:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:49:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:49:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_13 2026-02-20T21:49:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:49:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:49:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:49:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:49:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=8 2026-02-20T21:49:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:49:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:49:59.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 24373248 bytes 2026-02-20T21:49:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 90838016 bytes 2026-02-20T21:49:59.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 151962624 bytes 2026-02-20T21:49:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 169265152 bytes 2026-02-20T21:49:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 210355200 bytes 2026-02-20T21:50:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 245661696 bytes 2026-02-20T21:50:00.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 257804288 bytes 2026-02-20T21:50:00.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 328287232 bytes 2026-02-20T21:50:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 373448704 bytes 2026-02-20T21:50:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 411091968 bytes 2026-02-20T21:50:00.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 447842304 bytes 2026-02-20T21:50:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 462556160 bytes 2026-02-20T21:50:00.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 488076288 bytes 2026-02-20T21:50:00.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 583262208 bytes 2026-02-20T21:50:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 623041536 bytes 2026-02-20T21:50:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 666927104 bytes 2026-02-20T21:50:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 694973440 bytes 2026-02-20T21:50:00.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 717421568 bytes 2026-02-20T21:50:00.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 763163648 bytes 2026-02-20T21:50:00.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 788775936 bytes 2026-02-20T21:50:00.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 870576128 bytes 2026-02-20T21:50:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 874088448 bytes 2026-02-20T21:50:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 909085696 bytes 2026-02-20T21:50:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 999361536 bytes 2026-02-20T21:50:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1079699456 bytes 2026-02-20T21:50:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1130629120 bytes 2026-02-20T21:50:01.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1147961344 bytes 2026-02-20T21:50:01.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1184808960 bytes 2026-02-20T21:50:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1195633664 bytes 2026-02-20T21:50:01.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1304043520 bytes 2026-02-20T21:50:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1355052032 bytes 2026-02-20T21:50:01.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1381064704 bytes 2026-02-20T21:50:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1415367680 bytes 2026-02-20T21:50:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1433737216 bytes 2026-02-20T21:50:01.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1483674624 bytes 2026-02-20T21:50:01.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1592645632 bytes 2026-02-20T21:50:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1621988352 bytes 2026-02-20T21:50:01.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1662047232 bytes 2026-02-20T21:50:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1687980032 bytes 2026-02-20T21:50:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1733073920 bytes 2026-02-20T21:50:01.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1763602432 bytes 2026-02-20T21:50:01.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1785147392 bytes 2026-02-20T21:50:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1826461696 bytes 2026-02-20T21:50:01.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1837924352 bytes 2026-02-20T21:50:01.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1892534272 bytes 2026-02-20T21:50:01.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1912916992 bytes 2026-02-20T21:50:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2010536960 bytes 2026-02-20T21:50:02.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2048440320 bytes 2026-02-20T21:50:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2131284992 bytes 2026-02-20T21:50:02.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2199959552 bytes 2026-02-20T21:50:02.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2301343744 bytes 2026-02-20T21:50:02.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2328612864 bytes 2026-02-20T21:50:02.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2389535744 bytes 2026-02-20T21:50:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2410060800 bytes 2026-02-20T21:50:02.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2448661504 bytes 2026-02-20T21:50:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2469637120 bytes 2026-02-20T21:50:02.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2500400128 bytes 2026-02-20T21:50:02.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2573199360 bytes 2026-02-20T21:50:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2574266368 bytes 2026-02-20T21:50:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2581042176 bytes 2026-02-20T21:50:02.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2609115136 bytes 2026-02-20T21:50:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2632125440 bytes 2026-02-20T21:50:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2687005696 bytes 2026-02-20T21:50:02.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2687609856 bytes 2026-02-20T21:50:02.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2693090304 bytes 2026-02-20T21:50:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2712651776 bytes 2026-02-20T21:50:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2757737472 bytes 2026-02-20T21:50:02.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2780443648 bytes 2026-02-20T21:50:02.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2846858240 bytes 2026-02-20T21:50:03.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2892259328 bytes 2026-02-20T21:50:03.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2915911680 bytes 2026-02-20T21:50:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2924503040 bytes 2026-02-20T21:50:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3013395456 bytes 2026-02-20T21:50:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3056229376 bytes 2026-02-20T21:50:03.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3093365760 bytes 2026-02-20T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3143950336 bytes 2026-02-20T21:50:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3166418944 bytes 2026-02-20T21:50:03.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3210776576 bytes 2026-02-20T21:50:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3236469760 bytes 2026-02-20T21:50:03.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3259351040 bytes 2026-02-20T21:50:03.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3299688448 bytes 2026-02-20T21:50:03.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3354663936 bytes 2026-02-20T21:50:03.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3391432704 bytes 2026-02-20T21:50:03.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3430507520 bytes 2026-02-20T21:50:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3432861696 bytes 2026-02-20T21:50:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3488607232 bytes 2026-02-20T21:50:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3542067200 bytes 2026-02-20T21:50:03.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3579009024 bytes 2026-02-20T21:50:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3594808320 bytes 2026-02-20T21:50:03.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3672916992 bytes 2026-02-20T21:50:03.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3702127616 bytes 2026-02-20T21:50:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3762809856 bytes 2026-02-20T21:50:04.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3763396608 bytes 2026-02-20T21:50:04.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3805734912 bytes 2026-02-20T21:50:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3841247232 bytes 2026-02-20T21:50:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3841878016 bytes 2026-02-20T21:50:04.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3880064000 bytes 2026-02-20T21:50:04.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3929742336 bytes 2026-02-20T21:50:04.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3947516928 bytes 2026-02-20T21:50:04.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3974969344 bytes 2026-02-20T21:50:04.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 3989695488 bytes 2026-02-20T21:50:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 3995268096 bytes 2026-02-20T21:50:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4039933952 bytes 2026-02-20T21:50:04.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4112241664 bytes 2026-02-20T21:50:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4148427776 bytes 2026-02-20T21:50:04.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4173716480 bytes 2026-02-20T21:50:04.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4214080512 bytes 2026-02-20T21:50:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4261055488 bytes 2026-02-20T21:50:04.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4291849216 bytes 2026-02-20T21:50:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4310386688 bytes 2026-02-20T21:50:05.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4334844928 bytes 2026-02-20T21:50:05.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4368305152 bytes 2026-02-20T21:50:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4429127680 bytes 2026-02-20T21:50:05.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4494132224 bytes 2026-02-20T21:50:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4517448704 bytes 2026-02-20T21:50:05.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4570643456 bytes 2026-02-20T21:50:05.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4604981248 bytes 2026-02-20T21:50:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4641690624 bytes 2026-02-20T21:50:05.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4672103424 bytes 2026-02-20T21:50:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4749468672 bytes 2026-02-20T21:50:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4801846272 bytes 2026-02-20T21:50:05.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4851539968 bytes 2026-02-20T21:50:05.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4913568768 bytes 2026-02-20T21:50:05.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4961428480 bytes 2026-02-20T21:50:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4977136640 bytes 2026-02-20T21:50:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5011115008 bytes 2026-02-20T21:50:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5068206080 bytes 2026-02-20T21:50:05.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5081181184 bytes 2026-02-20T21:50:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5129871360 bytes 2026-02-20T21:50:06.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5212078080 bytes 2026-02-20T21:50:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5242459136 bytes 2026-02-20T21:50:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5264230400 bytes 2026-02-20T21:50:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5271752704 bytes 2026-02-20T21:50:06.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5308909568 bytes 2026-02-20T21:50:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5372003328 bytes 2026-02-20T21:50:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5410900992 bytes 2026-02-20T21:50:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5460799488 bytes 2026-02-20T21:50:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5523052544 bytes 2026-02-20T21:50:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5563035648 bytes 2026-02-20T21:50:06.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5616008192 bytes 2026-02-20T21:50:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5641352192 bytes 2026-02-20T21:50:06.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5651027968 bytes 2026-02-20T21:50:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5656826880 bytes 2026-02-20T21:50:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5713280000 bytes 2026-02-20T21:50:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5791206400 bytes 2026-02-20T21:50:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5809759232 bytes 2026-02-20T21:50:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5837511680 bytes 2026-02-20T21:50:06.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5838137344 bytes 2026-02-20T21:50:06.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5852250112 bytes 2026-02-20T21:50:06.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5914191872 bytes 2026-02-20T21:50:06.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 5955333120 bytes 2026-02-20T21:50:06.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6015399936 bytes 2026-02-20T21:50:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6075568128 bytes 2026-02-20T21:50:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6092502016 bytes 2026-02-20T21:50:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6142526464 bytes 2026-02-20T21:50:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6215668736 bytes 2026-02-20T21:50:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6235315200 bytes 2026-02-20T21:50:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6283329536 bytes 2026-02-20T21:50:07.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6367733760 bytes 2026-02-20T21:50:07.384 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6420731904 bytes 2026-02-20T21:50:07.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6466659328 bytes 2026-02-20T21:50:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6506479616 bytes 2026-02-20T21:50:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6543362048 bytes 2026-02-20T21:50:07.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6611969024 bytes 2026-02-20T21:50:07.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6630609920 bytes 2026-02-20T21:50:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6695061504 bytes 2026-02-20T21:50:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:50:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:50:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:50:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:50:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_14 2026-02-20T21:50:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 snapshot 128M 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_14 128M 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:50:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:50:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:50:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:50:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_14 2026-02-20T21:50:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:50:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:50:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_14 snapshot 2026-02-20T21:50:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:50:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_14 snapshot 2026-02-20T21:50:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:50:08.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_14 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_14': 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 1314b97dcdca 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.1314b97dcdca 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:07 2026 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:07 2026 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:07 2026 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 14383d26-7af3-40c8-90a8-27603ddc790c 2026-02-20T21:50:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:50:08.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:50:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_14 2026-02-20T21:50:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:50:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:50:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=11 2026-02-20T21:50:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:50:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:50:08.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 74803200 bytes 2026-02-20T21:50:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 93702144 bytes 2026-02-20T21:50:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 140999680 bytes 2026-02-20T21:50:08.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 164647936 bytes 2026-02-20T21:50:09.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 184696832 bytes 2026-02-20T21:50:09.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 188607488 bytes 2026-02-20T21:50:09.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 227147776 bytes 2026-02-20T21:50:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 296698880 bytes 2026-02-20T21:50:09.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 324969472 bytes 2026-02-20T21:50:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 345126912 bytes 2026-02-20T21:50:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 386454528 bytes 2026-02-20T21:50:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 405638144 bytes 2026-02-20T21:50:09.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 524696576 bytes 2026-02-20T21:50:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 580247552 bytes 2026-02-20T21:50:09.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 648944640 bytes 2026-02-20T21:50:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 689635328 bytes 2026-02-20T21:50:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 780490752 bytes 2026-02-20T21:50:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 826875904 bytes 2026-02-20T21:50:09.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 861346816 bytes 2026-02-20T21:50:09.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 882466816 bytes 2026-02-20T21:50:09.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 906703872 bytes 2026-02-20T21:50:09.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 951531520 bytes 2026-02-20T21:50:09.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1038119936 bytes 2026-02-20T21:50:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1048704000 bytes 2026-02-20T21:50:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1065469952 bytes 2026-02-20T21:50:09.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1119884288 bytes 2026-02-20T21:50:10.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1152399360 bytes 2026-02-20T21:50:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1245626368 bytes 2026-02-20T21:50:10.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1275423744 bytes 2026-02-20T21:50:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1390310400 bytes 2026-02-20T21:50:10.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1461363712 bytes 2026-02-20T21:50:10.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1485663232 bytes 2026-02-20T21:50:10.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1512871936 bytes 2026-02-20T21:50:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1552509952 bytes 2026-02-20T21:50:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1610654720 bytes 2026-02-20T21:50:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1666217984 bytes 2026-02-20T21:50:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1683302400 bytes 2026-02-20T21:50:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1687888896 bytes 2026-02-20T21:50:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1750662144 bytes 2026-02-20T21:50:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1802365952 bytes 2026-02-20T21:50:10.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1848817664 bytes 2026-02-20T21:50:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1864069120 bytes 2026-02-20T21:50:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1878791168 bytes 2026-02-20T21:50:11.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1938301952 bytes 2026-02-20T21:50:11.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2022033408 bytes 2026-02-20T21:50:11.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2052563968 bytes 2026-02-20T21:50:11.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2142141440 bytes 2026-02-20T21:50:11.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2175042560 bytes 2026-02-20T21:50:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2175683584 bytes 2026-02-20T21:50:11.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2204200960 bytes 2026-02-20T21:50:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2261663744 bytes 2026-02-20T21:50:11.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2269595648 bytes 2026-02-20T21:50:11.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2275378176 bytes 2026-02-20T21:50:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2322416640 bytes 2026-02-20T21:50:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2343687168 bytes 2026-02-20T21:50:11.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2355225600 bytes 2026-02-20T21:50:11.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2392856576 bytes 2026-02-20T21:50:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2457412608 bytes 2026-02-20T21:50:11.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2458027008 bytes 2026-02-20T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2503506944 bytes 2026-02-20T21:50:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2584081408 bytes 2026-02-20T21:50:11.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2612882432 bytes 2026-02-20T21:50:11.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2641891328 bytes 2026-02-20T21:50:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2695278592 bytes 2026-02-20T21:50:12.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2739398656 bytes 2026-02-20T21:50:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2802931712 bytes 2026-02-20T21:50:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2814586880 bytes 2026-02-20T21:50:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2822158336 bytes 2026-02-20T21:50:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2822737920 bytes 2026-02-20T21:50:12.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2876827648 bytes 2026-02-20T21:50:12.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2877451264 bytes 2026-02-20T21:50:12.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2909239296 bytes 2026-02-20T21:50:12.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2993102848 bytes 2026-02-20T21:50:12.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3021776896 bytes 2026-02-20T21:50:12.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3034907648 bytes 2026-02-20T21:50:12.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3060345856 bytes 2026-02-20T21:50:12.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3086714880 bytes 2026-02-20T21:50:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3089501184 bytes 2026-02-20T21:50:12.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3149093888 bytes 2026-02-20T21:50:12.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3219800064 bytes 2026-02-20T21:50:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3233007616 bytes 2026-02-20T21:50:12.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3256684544 bytes 2026-02-20T21:50:12.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3322420224 bytes 2026-02-20T21:50:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3395542016 bytes 2026-02-20T21:50:12.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3472139264 bytes 2026-02-20T21:50:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3479984128 bytes 2026-02-20T21:50:12.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3481842688 bytes 2026-02-20T21:50:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3496607744 bytes 2026-02-20T21:50:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3583993856 bytes 2026-02-20T21:50:13.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3633635328 bytes 2026-02-20T21:50:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3653055488 bytes 2026-02-20T21:50:13.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3682088960 bytes 2026-02-20T21:50:13.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3682712576 bytes 2026-02-20T21:50:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3763264512 bytes 2026-02-20T21:50:13.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3806779392 bytes 2026-02-20T21:50:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3807383552 bytes 2026-02-20T21:50:13.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3839984640 bytes 2026-02-20T21:50:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3916311552 bytes 2026-02-20T21:50:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3928292352 bytes 2026-02-20T21:50:13.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3953623040 bytes 2026-02-20T21:50:13.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 3990703104 bytes 2026-02-20T21:50:13.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4000675840 bytes 2026-02-20T21:50:13.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4003903488 bytes 2026-02-20T21:50:13.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4012208128 bytes 2026-02-20T21:50:13.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4032623616 bytes 2026-02-20T21:50:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4054045696 bytes 2026-02-20T21:50:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4071297024 bytes 2026-02-20T21:50:13.825 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:50:13.821+0000 7fa5e2658640 -1 reset not still connected to 0x55b902d84270 2026-02-20T21:50:13.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4140772352 bytes 2026-02-20T21:50:13.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4191382528 bytes 2026-02-20T21:50:13.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4278181888 bytes 2026-02-20T21:50:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4332063744 bytes 2026-02-20T21:50:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4344555520 bytes 2026-02-20T21:50:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4373800960 bytes 2026-02-20T21:50:14.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4424868864 bytes 2026-02-20T21:50:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4504658944 bytes 2026-02-20T21:50:14.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4508226560 bytes 2026-02-20T21:50:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4516818944 bytes 2026-02-20T21:50:14.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4577156096 bytes 2026-02-20T21:50:14.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4617319424 bytes 2026-02-20T21:50:14.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4697361408 bytes 2026-02-20T21:50:14.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4732477440 bytes 2026-02-20T21:50:14.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4775076864 bytes 2026-02-20T21:50:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4775675904 bytes 2026-02-20T21:50:14.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4794741760 bytes 2026-02-20T21:50:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4847401984 bytes 2026-02-20T21:50:14.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 4889175040 bytes 2026-02-20T21:50:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 4905226240 bytes 2026-02-20T21:50:14.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 4969862144 bytes 2026-02-20T21:50:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5017005056 bytes 2026-02-20T21:50:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5040575488 bytes 2026-02-20T21:50:14.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5078202368 bytes 2026-02-20T21:50:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5175017472 bytes 2026-02-20T21:50:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5200482304 bytes 2026-02-20T21:50:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5272982528 bytes 2026-02-20T21:50:15.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5327998976 bytes 2026-02-20T21:50:15.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5334069248 bytes 2026-02-20T21:50:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5392417792 bytes 2026-02-20T21:50:15.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5477654528 bytes 2026-02-20T21:50:15.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5494240256 bytes 2026-02-20T21:50:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5552919552 bytes 2026-02-20T21:50:15.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5609390080 bytes 2026-02-20T21:50:15.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5682743296 bytes 2026-02-20T21:50:15.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5683355648 bytes 2026-02-20T21:50:15.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5743184896 bytes 2026-02-20T21:50:15.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5768182784 bytes 2026-02-20T21:50:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5816224768 bytes 2026-02-20T21:50:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5816851456 bytes 2026-02-20T21:50:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5854914560 bytes 2026-02-20T21:50:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5882887168 bytes 2026-02-20T21:50:15.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5920025600 bytes 2026-02-20T21:50:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 5992142848 bytes 2026-02-20T21:50:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6039821312 bytes 2026-02-20T21:50:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6115873792 bytes 2026-02-20T21:50:16.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6130255872 bytes 2026-02-20T21:50:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6146902016 bytes 2026-02-20T21:50:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6217692160 bytes 2026-02-20T21:50:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6247317504 bytes 2026-02-20T21:50:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6338435072 bytes 2026-02-20T21:50:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6352244736 bytes 2026-02-20T21:50:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6368319488 bytes 2026-02-20T21:50:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6413220864 bytes 2026-02-20T21:50:16.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6486460416 bytes 2026-02-20T21:50:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6501588992 bytes 2026-02-20T21:50:16.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6535588864 bytes 2026-02-20T21:50:16.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6583838720 bytes 2026-02-20T21:50:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6618113024 bytes 2026-02-20T21:50:16.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6691197952 bytes 2026-02-20T21:50:16.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6769143808 bytes 2026-02-20T21:50:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6869745664 bytes 2026-02-20T21:50:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6955495424 bytes 2026-02-20T21:50:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6975965184 bytes 2026-02-20T21:50:16.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6992952320 bytes 2026-02-20T21:50:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7033203712 bytes 2026-02-20T21:50:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7060077568 bytes 2026-02-20T21:50:17.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7095194624 bytes 2026-02-20T21:50:17.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7110337536 bytes 2026-02-20T21:50:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7182585856 bytes 2026-02-20T21:50:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7221511168 bytes 2026-02-20T21:50:17.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7291459584 bytes 2026-02-20T21:50:17.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7302937600 bytes 2026-02-20T21:50:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7317738496 bytes 2026-02-20T21:50:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7343254528 bytes 2026-02-20T21:50:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7399116800 bytes 2026-02-20T21:50:17.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7414379520 bytes 2026-02-20T21:50:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7458168832 bytes 2026-02-20T21:50:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7501985792 bytes 2026-02-20T21:50:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7521415168 bytes 2026-02-20T21:50:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7600890880 bytes 2026-02-20T21:50:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7640583168 bytes 2026-02-20T21:50:17.983 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8249484608427279) > weight (0.25) 2026-02-20T21:50:17.983 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-20T21:50:17.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-20T21:50:17.983 INFO:tasks.rbd_mirror_thrash:waiting for 5.920213816640734 secs before reviving daemons 2026-02-20T21:50:17.983 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T21:50:17.980+0000 7fd1a308a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 28241) UID: 1000 2026-02-20T21:50:17.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7702605824 bytes 2026-02-20T21:50:18.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7777119232 bytes 2026-02-20T21:50:18.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7874944000 bytes 2026-02-20T21:50:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7939156992 bytes 2026-02-20T21:50:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 7954114560 bytes 2026-02-20T21:50:18.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8000419840 bytes 2026-02-20T21:50:18.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8024011776 bytes 2026-02-20T21:50:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8050636800 bytes 2026-02-20T21:50:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8139819008 bytes 2026-02-20T21:50:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8154205184 bytes 2026-02-20T21:50:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8203247616 bytes 2026-02-20T21:50:18.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8258072576 bytes 2026-02-20T21:50:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8293470208 bytes 2026-02-20T21:50:18.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8316847104 bytes 2026-02-20T21:50:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8334894080 bytes 2026-02-20T21:50:18.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8389223424 bytes 2026-02-20T21:50:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8454786048 bytes 2026-02-20T21:50:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8455412736 bytes 2026-02-20T21:50:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8511735808 bytes 2026-02-20T21:50:18.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8568499200 bytes 2026-02-20T21:50:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8604063744 bytes 2026-02-20T21:50:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8693404672 bytes 2026-02-20T21:50:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8708669440 bytes 2026-02-20T21:50:19.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8803998720 bytes 2026-02-20T21:50:19.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8866992128 bytes 2026-02-20T21:50:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8898997248 bytes 2026-02-20T21:50:19.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 8959367168 bytes 2026-02-20T21:50:19.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 8985919488 bytes 2026-02-20T21:50:19.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9031123968 bytes 2026-02-20T21:50:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9065648128 bytes 2026-02-20T21:50:19.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9068297216 bytes 2026-02-20T21:50:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9146537984 bytes 2026-02-20T21:50:19.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9169754112 bytes 2026-02-20T21:50:19.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9201021952 bytes 2026-02-20T21:50:19.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9226012672 bytes 2026-02-20T21:50:19.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9259853824 bytes 2026-02-20T21:50:19.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_15 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 snapshot 128M 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:19.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_15 128M 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:19.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:50:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:50:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:50:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:50:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:19.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_15 2026-02-20T21:50:19.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:50:19.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:19.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:50:19.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:19.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_15 snapshot 2026-02-20T21:50:19.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:19.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:19.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:50:19.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:19.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_15 snapshot 2026-02-20T21:50:20.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:50:20.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_15 2026-02-20T21:50:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_15': 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 1320bf2de6d1 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.1320bf2de6d1 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:50:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:19 2026 2026-02-20T21:50:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:19 2026 2026-02-20T21:50:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:19 2026 2026-02-20T21:50:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:50:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:50:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: d173de17-b1d5-4604-94f9-5797e13957df 2026-02-20T21:50:20.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:50:20.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:50:20.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_15 2026-02-20T21:50:20.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:20.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:20.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:50:20.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:50:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:50:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:50:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:50:20.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 29336576 bytes 2026-02-20T21:50:20.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 102088704 bytes 2026-02-20T21:50:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 161471488 bytes 2026-02-20T21:50:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 253376512 bytes 2026-02-20T21:50:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 254652416 bytes 2026-02-20T21:50:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 334422016 bytes 2026-02-20T21:50:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 356835328 bytes 2026-02-20T21:50:21.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 358599680 bytes 2026-02-20T21:50:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 431637504 bytes 2026-02-20T21:50:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 535875584 bytes 2026-02-20T21:50:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 560617472 bytes 2026-02-20T21:50:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 601896960 bytes 2026-02-20T21:50:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 652349440 bytes 2026-02-20T21:50:21.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 686353408 bytes 2026-02-20T21:50:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 752580608 bytes 2026-02-20T21:50:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 793485312 bytes 2026-02-20T21:50:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 862233600 bytes 2026-02-20T21:50:21.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 946955264 bytes 2026-02-20T21:50:21.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 978328576 bytes 2026-02-20T21:50:21.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1020748800 bytes 2026-02-20T21:50:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1046160384 bytes 2026-02-20T21:50:21.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1089093632 bytes 2026-02-20T21:50:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1144195072 bytes 2026-02-20T21:50:22.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1186276352 bytes 2026-02-20T21:50:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1242988544 bytes 2026-02-20T21:50:22.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1308461056 bytes 2026-02-20T21:50:22.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1373641728 bytes 2026-02-20T21:50:22.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1409823744 bytes 2026-02-20T21:50:22.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1460006912 bytes 2026-02-20T21:50:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1478654976 bytes 2026-02-20T21:50:22.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1516172288 bytes 2026-02-20T21:50:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1561840640 bytes 2026-02-20T21:50:22.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1593419776 bytes 2026-02-20T21:50:22.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1608019968 bytes 2026-02-20T21:50:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1647921152 bytes 2026-02-20T21:50:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1686618112 bytes 2026-02-20T21:50:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1750813696 bytes 2026-02-20T21:50:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1811532800 bytes 2026-02-20T21:50:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1882554368 bytes 2026-02-20T21:50:22.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1929532416 bytes 2026-02-20T21:50:22.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 2012412928 bytes 2026-02-20T21:50:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2065628160 bytes 2026-02-20T21:50:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2090403840 bytes 2026-02-20T21:50:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2098686976 bytes 2026-02-20T21:50:23.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2117318656 bytes 2026-02-20T21:50:23.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2202619904 bytes 2026-02-20T21:50:23.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2217798656 bytes 2026-02-20T21:50:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2286260224 bytes 2026-02-20T21:50:23.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2295608320 bytes 2026-02-20T21:50:23.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2318548992 bytes 2026-02-20T21:50:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2333741056 bytes 2026-02-20T21:50:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2381092864 bytes 2026-02-20T21:50:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2407311360 bytes 2026-02-20T21:50:23.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2500954112 bytes 2026-02-20T21:50:23.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2608280576 bytes 2026-02-20T21:50:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2685386752 bytes 2026-02-20T21:50:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2688967680 bytes 2026-02-20T21:50:23.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2739456000 bytes 2026-02-20T21:50:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2801379328 bytes 2026-02-20T21:50:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2831356928 bytes 2026-02-20T21:50:23.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2843693056 bytes 2026-02-20T21:50:23.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2855014400 bytes 2026-02-20T21:50:23.904 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-20T21:50:23.904 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:50:23.904 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-20T21:50:23.904 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T21:50:23.905 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T21:50:23.906 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T21:50:23.907 INFO:tasks.rbd_mirror_thrash:waiting for 75.28541520198128 secs before thrashing 2026-02-20T21:50:23.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2945696768 bytes 2026-02-20T21:50:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2960756736 bytes 2026-02-20T21:50:23.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2961362944 bytes 2026-02-20T21:50:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3024920576 bytes 2026-02-20T21:50:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3079237632 bytes 2026-02-20T21:50:24.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3093619712 bytes 2026-02-20T21:50:24.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3171512320 bytes 2026-02-20T21:50:24.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3240558592 bytes 2026-02-20T21:50:24.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3241956352 bytes 2026-02-20T21:50:24.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3281972224 bytes 2026-02-20T21:50:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3318568960 bytes 2026-02-20T21:50:24.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3366162432 bytes 2026-02-20T21:50:24.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3370297344 bytes 2026-02-20T21:50:24.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3394603008 bytes 2026-02-20T21:50:24.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3408640000 bytes 2026-02-20T21:50:24.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3434437632 bytes 2026-02-20T21:50:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3453884416 bytes 2026-02-20T21:50:24.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3560782848 bytes 2026-02-20T21:50:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3603110912 bytes 2026-02-20T21:50:24.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3649130496 bytes 2026-02-20T21:50:24.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3743633408 bytes 2026-02-20T21:50:24.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3817339904 bytes 2026-02-20T21:50:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3851369472 bytes 2026-02-20T21:50:25.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3907588096 bytes 2026-02-20T21:50:25.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3912865792 bytes 2026-02-20T21:50:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3947860992 bytes 2026-02-20T21:50:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3971160064 bytes 2026-02-20T21:50:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3971783680 bytes 2026-02-20T21:50:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3994227712 bytes 2026-02-20T21:50:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4058796032 bytes 2026-02-20T21:50:25.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4079753216 bytes 2026-02-20T21:50:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4080822272 bytes 2026-02-20T21:50:25.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4118700032 bytes 2026-02-20T21:50:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4137668608 bytes 2026-02-20T21:50:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4153224192 bytes 2026-02-20T21:50:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4170514432 bytes 2026-02-20T21:50:25.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4212859904 bytes 2026-02-20T21:50:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4241879040 bytes 2026-02-20T21:50:25.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4242504704 bytes 2026-02-20T21:50:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4298525696 bytes 2026-02-20T21:50:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4318695424 bytes 2026-02-20T21:50:25.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4351828992 bytes 2026-02-20T21:50:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4388308992 bytes 2026-02-20T21:50:25.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4401316864 bytes 2026-02-20T21:50:25.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4410917888 bytes 2026-02-20T21:50:25.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4424702976 bytes 2026-02-20T21:50:25.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4444081152 bytes 2026-02-20T21:50:25.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4461107200 bytes 2026-02-20T21:50:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4538609664 bytes 2026-02-20T21:50:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4614717440 bytes 2026-02-20T21:50:26.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4675630080 bytes 2026-02-20T21:50:26.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4696022016 bytes 2026-02-20T21:50:26.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4732220416 bytes 2026-02-20T21:50:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4779831296 bytes 2026-02-20T21:50:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4829075456 bytes 2026-02-20T21:50:26.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4879723520 bytes 2026-02-20T21:50:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4999775232 bytes 2026-02-20T21:50:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5084239872 bytes 2026-02-20T21:50:26.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5094226944 bytes 2026-02-20T21:50:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5161710592 bytes 2026-02-20T21:50:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5250055168 bytes 2026-02-20T21:50:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5293578240 bytes 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_16 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 snapshot 128M 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:26.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_16 128M 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_16 2026-02-20T21:50:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:50:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:50:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_16 snapshot 2026-02-20T21:50:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:50:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_16 snapshot 2026-02-20T21:50:26.864 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7383113294682828) > weight (0.25) 2026-02-20T21:50:26.864 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.37594170418796236) > weight (0.25) 2026-02-20T21:50:26.864 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7154703649724611) > weight (0.25) 2026-02-20T21:50:26.864 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8112043290617567) > weight (0.25) 2026-02-20T21:50:26.864 INFO:tasks.rbd_mirror_thrash:waiting for 118.76711374092457 secs before thrashing 2026-02-20T21:50:27.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:50:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_16 2026-02-20T21:50:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_16': 2026-02-20T21:50:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13315fb9147d 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13315fb9147d 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:26 2026 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:26 2026 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:26 2026 2026-02-20T21:50:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:50:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:50:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: f3776a65-963f-4287-81b8-3f98573893e1 2026-02-20T21:50:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:50:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:50:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_16 2026-02-20T21:50:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:50:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:50:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=10 2026-02-20T21:50:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:50:27.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:50:27.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 20018176 bytes 2026-02-20T21:50:27.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 52466688 bytes 2026-02-20T21:50:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 116793344 bytes 2026-02-20T21:50:27.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 155141120 bytes 2026-02-20T21:50:28.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 254868480 bytes 2026-02-20T21:50:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 255469568 bytes 2026-02-20T21:50:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 310827008 bytes 2026-02-20T21:50:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 326272000 bytes 2026-02-20T21:50:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 372215808 bytes 2026-02-20T21:50:28.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 413978624 bytes 2026-02-20T21:50:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 463843328 bytes 2026-02-20T21:50:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 494442496 bytes 2026-02-20T21:50:28.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 521043968 bytes 2026-02-20T21:50:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 585153536 bytes 2026-02-20T21:50:28.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 658946048 bytes 2026-02-20T21:50:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 698992640 bytes 2026-02-20T21:50:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 736147456 bytes 2026-02-20T21:50:28.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 760718336 bytes 2026-02-20T21:50:28.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 809893888 bytes 2026-02-20T21:50:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 827100160 bytes 2026-02-20T21:50:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 885401600 bytes 2026-02-20T21:50:28.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 913481728 bytes 2026-02-20T21:50:28.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 939213824 bytes 2026-02-20T21:50:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 948525056 bytes 2026-02-20T21:50:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 978035712 bytes 2026-02-20T21:50:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 991066112 bytes 2026-02-20T21:50:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1007301632 bytes 2026-02-20T21:50:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1067422720 bytes 2026-02-20T21:50:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1126245376 bytes 2026-02-20T21:50:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1164500992 bytes 2026-02-20T21:50:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1182966784 bytes 2026-02-20T21:50:29.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1227681792 bytes 2026-02-20T21:50:29.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1286064128 bytes 2026-02-20T21:50:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1331125248 bytes 2026-02-20T21:50:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1390445568 bytes 2026-02-20T21:50:29.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1407533056 bytes 2026-02-20T21:50:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1408149504 bytes 2026-02-20T21:50:29.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1474912256 bytes 2026-02-20T21:50:29.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1539767296 bytes 2026-02-20T21:50:29.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1573120000 bytes 2026-02-20T21:50:29.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1648803840 bytes 2026-02-20T21:50:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1713213440 bytes 2026-02-20T21:50:29.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1716664320 bytes 2026-02-20T21:50:29.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1727783936 bytes 2026-02-20T21:50:29.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1728524288 bytes 2026-02-20T21:50:29.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1745765376 bytes 2026-02-20T21:50:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1756626944 bytes 2026-02-20T21:50:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1810597888 bytes 2026-02-20T21:50:30.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1833730048 bytes 2026-02-20T21:50:30.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1865602048 bytes 2026-02-20T21:50:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 1936004096 bytes 2026-02-20T21:50:30.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2019000320 bytes 2026-02-20T21:50:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2063845376 bytes 2026-02-20T21:50:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2100692992 bytes 2026-02-20T21:50:30.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2114104320 bytes 2026-02-20T21:50:30.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2174533632 bytes 2026-02-20T21:50:30.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2188189696 bytes 2026-02-20T21:50:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2208689152 bytes 2026-02-20T21:50:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2237748224 bytes 2026-02-20T21:50:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2272641024 bytes 2026-02-20T21:50:30.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2273264640 bytes 2026-02-20T21:50:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2295288832 bytes 2026-02-20T21:50:30.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2307110912 bytes 2026-02-20T21:50:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2333982720 bytes 2026-02-20T21:50:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2370196480 bytes 2026-02-20T21:50:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2419661824 bytes 2026-02-20T21:50:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2465466368 bytes 2026-02-20T21:50:31.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2503759872 bytes 2026-02-20T21:50:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2504398848 bytes 2026-02-20T21:50:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2565554176 bytes 2026-02-20T21:50:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2636382208 bytes 2026-02-20T21:50:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2644626432 bytes 2026-02-20T21:50:31.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2701254656 bytes 2026-02-20T21:50:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2746613760 bytes 2026-02-20T21:50:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2785863680 bytes 2026-02-20T21:50:31.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 2809843712 bytes 2026-02-20T21:50:31.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 2839004160 bytes 2026-02-20T21:50:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 2882685952 bytes 2026-02-20T21:50:31.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 2925239296 bytes 2026-02-20T21:50:31.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3007620096 bytes 2026-02-20T21:50:31.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3029585920 bytes 2026-02-20T21:50:31.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3049638912 bytes 2026-02-20T21:50:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3124752384 bytes 2026-02-20T21:50:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3174426624 bytes 2026-02-20T21:50:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3197631488 bytes 2026-02-20T21:50:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3208837120 bytes 2026-02-20T21:50:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3219323904 bytes 2026-02-20T21:50:31.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3262439424 bytes 2026-02-20T21:50:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3290806272 bytes 2026-02-20T21:50:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3316283392 bytes 2026-02-20T21:50:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3369019392 bytes 2026-02-20T21:50:32.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3408066560 bytes 2026-02-20T21:50:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3473567744 bytes 2026-02-20T21:50:32.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3497622528 bytes 2026-02-20T21:50:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3555267584 bytes 2026-02-20T21:50:32.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3593720832 bytes 2026-02-20T21:50:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3615219712 bytes 2026-02-20T21:50:32.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3638306816 bytes 2026-02-20T21:50:32.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3677179904 bytes 2026-02-20T21:50:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3738148864 bytes 2026-02-20T21:50:32.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 3783425024 bytes 2026-02-20T21:50:32.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 3836341248 bytes 2026-02-20T21:50:32.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 3858035712 bytes 2026-02-20T21:50:32.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 3858665472 bytes 2026-02-20T21:50:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 3892387840 bytes 2026-02-20T21:50:32.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 3916318720 bytes 2026-02-20T21:50:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 3962226688 bytes 2026-02-20T21:50:32.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4048755712 bytes 2026-02-20T21:50:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4068323328 bytes 2026-02-20T21:50:33.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4080834560 bytes 2026-02-20T21:50:33.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4089362432 bytes 2026-02-20T21:50:33.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4102395904 bytes 2026-02-20T21:50:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4110936064 bytes 2026-02-20T21:50:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4145033216 bytes 2026-02-20T21:50:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4163721216 bytes 2026-02-20T21:50:33.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4227503104 bytes 2026-02-20T21:50:33.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4259329024 bytes 2026-02-20T21:50:33.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4332748800 bytes 2026-02-20T21:50:33.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4382373888 bytes 2026-02-20T21:50:33.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4435871744 bytes 2026-02-20T21:50:33.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4480165888 bytes 2026-02-20T21:50:33.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4543091712 bytes 2026-02-20T21:50:33.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4581100544 bytes 2026-02-20T21:50:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4656135168 bytes 2026-02-20T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4685895680 bytes 2026-02-20T21:50:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 4707832832 bytes 2026-02-20T21:50:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 4802547712 bytes 2026-02-20T21:50:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 4846917632 bytes 2026-02-20T21:50:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 4973270016 bytes 2026-02-20T21:50:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 4992140288 bytes 2026-02-20T21:50:34.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5023828992 bytes 2026-02-20T21:50:34.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5059314688 bytes 2026-02-20T21:50:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5067285504 bytes 2026-02-20T21:50:34.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5074148352 bytes 2026-02-20T21:50:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5132142592 bytes 2026-02-20T21:50:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5156036608 bytes 2026-02-20T21:50:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5173107712 bytes 2026-02-20T21:50:34.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5201510400 bytes 2026-02-20T21:50:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5266210816 bytes 2026-02-20T21:50:34.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5267639296 bytes 2026-02-20T21:50:34.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5348570112 bytes 2026-02-20T21:50:34.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5405808640 bytes 2026-02-20T21:50:34.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5414119424 bytes 2026-02-20T21:50:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5481367552 bytes 2026-02-20T21:50:34.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5501209600 bytes 2026-02-20T21:50:34.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5543652352 bytes 2026-02-20T21:50:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5581624320 bytes 2026-02-20T21:50:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5603511296 bytes 2026-02-20T21:50:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5639485440 bytes 2026-02-20T21:50:34.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5679925248 bytes 2026-02-20T21:50:35.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 5750241280 bytes 2026-02-20T21:50:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 5782112256 bytes 2026-02-20T21:50:35.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 5812328448 bytes 2026-02-20T21:50:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 5832211456 bytes 2026-02-20T21:50:35.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 5844062208 bytes 2026-02-20T21:50:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 5868486656 bytes 2026-02-20T21:50:35.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 5959744512 bytes 2026-02-20T21:50:35.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6030372864 bytes 2026-02-20T21:50:35.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6050022400 bytes 2026-02-20T21:50:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6113395712 bytes 2026-02-20T21:50:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6167910400 bytes 2026-02-20T21:50:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6198540288 bytes 2026-02-20T21:50:35.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6275618816 bytes 2026-02-20T21:50:35.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6324381696 bytes 2026-02-20T21:50:35.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6375433216 bytes 2026-02-20T21:50:35.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6407322624 bytes 2026-02-20T21:50:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6439929856 bytes 2026-02-20T21:50:35.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6511423488 bytes 2026-02-20T21:50:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6514843648 bytes 2026-02-20T21:50:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6563393536 bytes 2026-02-20T21:50:36.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6595383296 bytes 2026-02-20T21:50:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6669693952 bytes 2026-02-20T21:50:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 6688731136 bytes 2026-02-20T21:50:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 6743070720 bytes 2026-02-20T21:50:36.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 6754128896 bytes 2026-02-20T21:50:36.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 6806025216 bytes 2026-02-20T21:50:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 6859525120 bytes 2026-02-20T21:50:36.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 6914147328 bytes 2026-02-20T21:50:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 6925389824 bytes 2026-02-20T21:50:36.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7026682880 bytes 2026-02-20T21:50:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7097051136 bytes 2026-02-20T21:50:36.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7112744960 bytes 2026-02-20T21:50:36.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7133101056 bytes 2026-02-20T21:50:36.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7177468928 bytes 2026-02-20T21:50:36.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7178078208 bytes 2026-02-20T21:50:36.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7252375552 bytes 2026-02-20T21:50:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7298792448 bytes 2026-02-20T21:50:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7311172608 bytes 2026-02-20T21:50:36.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7345525760 bytes 2026-02-20T21:50:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7403424768 bytes 2026-02-20T21:50:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7442623488 bytes 2026-02-20T21:50:37.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7461652480 bytes 2026-02-20T21:50:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7486298112 bytes 2026-02-20T21:50:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 7486895104 bytes 2026-02-20T21:50:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 7588562944 bytes 2026-02-20T21:50:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 7643501568 bytes 2026-02-20T21:50:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 7667236864 bytes 2026-02-20T21:50:37.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 7746822144 bytes 2026-02-20T21:50:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 7836753920 bytes 2026-02-20T21:50:37.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 7904071680 bytes 2026-02-20T21:50:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 7974885376 bytes 2026-02-20T21:50:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 7975493632 bytes 2026-02-20T21:50:37.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8039054336 bytes 2026-02-20T21:50:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:50:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:50:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:50:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:50:37.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_17 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 snapshot 128M 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_17 128M 2026-02-20T21:50:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:37.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_17 2026-02-20T21:50:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:50:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:50:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_17 snapshot 2026-02-20T21:50:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:50:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:37.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_17 snapshot 2026-02-20T21:50:38.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:50:38.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_17 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_17': 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 133d72858a 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.133d72858a 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:50:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:50:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:37 2026 2026-02-20T21:50:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:37 2026 2026-02-20T21:50:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:37 2026 2026-02-20T21:50:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:50:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:50:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: e65dbfbb-2857-4359-9849-aa0a0d8627ec 2026-02-20T21:50:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:50:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:50:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_17 2026-02-20T21:50:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:50:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:50:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=10 2026-02-20T21:50:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:50:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:50:38.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 19273728 bytes 2026-02-20T21:50:38.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 64346112 bytes 2026-02-20T21:50:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 86080512 bytes 2026-02-20T21:50:38.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 155100160 bytes 2026-02-20T21:50:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 206562304 bytes 2026-02-20T21:50:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 236389376 bytes 2026-02-20T21:50:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 314382336 bytes 2026-02-20T21:50:39.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 322710528 bytes 2026-02-20T21:50:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 385011712 bytes 2026-02-20T21:50:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 400558080 bytes 2026-02-20T21:50:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 449854464 bytes 2026-02-20T21:50:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 493335552 bytes 2026-02-20T21:50:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 577480704 bytes 2026-02-20T21:50:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 593015808 bytes 2026-02-20T21:50:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 634175488 bytes 2026-02-20T21:50:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 679581696 bytes 2026-02-20T21:50:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 714878976 bytes 2026-02-20T21:50:39.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 738137088 bytes 2026-02-20T21:50:39.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 793851904 bytes 2026-02-20T21:50:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 876881920 bytes 2026-02-20T21:50:39.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 930279424 bytes 2026-02-20T21:50:39.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 948475904 bytes 2026-02-20T21:50:39.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 998590464 bytes 2026-02-20T21:50:39.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1025280000 bytes 2026-02-20T21:50:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1097988096 bytes 2026-02-20T21:50:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1122528256 bytes 2026-02-20T21:50:40.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1204003840 bytes 2026-02-20T21:50:40.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1298419712 bytes 2026-02-20T21:50:40.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1316661248 bytes 2026-02-20T21:50:40.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1329670144 bytes 2026-02-20T21:50:40.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1344053248 bytes 2026-02-20T21:50:40.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1431625728 bytes 2026-02-20T21:50:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1487382528 bytes 2026-02-20T21:50:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1513632768 bytes 2026-02-20T21:50:40.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1553888256 bytes 2026-02-20T21:50:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1562014720 bytes 2026-02-20T21:50:40.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1575357440 bytes 2026-02-20T21:50:40.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1578320896 bytes 2026-02-20T21:50:40.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1603067904 bytes 2026-02-20T21:50:40.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1633567744 bytes 2026-02-20T21:50:40.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1640475648 bytes 2026-02-20T21:50:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1697331200 bytes 2026-02-20T21:50:40.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1788536832 bytes 2026-02-20T21:50:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1845979136 bytes 2026-02-20T21:50:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1882864640 bytes 2026-02-20T21:50:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1895872512 bytes 2026-02-20T21:50:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1905906688 bytes 2026-02-20T21:50:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1961816064 bytes 2026-02-20T21:50:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1975334912 bytes 2026-02-20T21:50:41.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2027344896 bytes 2026-02-20T21:50:41.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2054762496 bytes 2026-02-20T21:50:41.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2089553920 bytes 2026-02-20T21:50:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2095171584 bytes 2026-02-20T21:50:41.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2105449472 bytes 2026-02-20T21:50:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2121994240 bytes 2026-02-20T21:50:41.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2151003136 bytes 2026-02-20T21:50:41.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2228594688 bytes 2026-02-20T21:50:41.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2291678208 bytes 2026-02-20T21:50:41.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2348731392 bytes 2026-02-20T21:50:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2366903296 bytes 2026-02-20T21:50:41.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2421099520 bytes 2026-02-20T21:50:41.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2534466560 bytes 2026-02-20T21:50:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2579158016 bytes 2026-02-20T21:50:41.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2601966592 bytes 2026-02-20T21:50:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2634656768 bytes 2026-02-20T21:50:41.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2664684544 bytes 2026-02-20T21:50:42.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2696668160 bytes 2026-02-20T21:50:42.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2697270272 bytes 2026-02-20T21:50:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2731790336 bytes 2026-02-20T21:50:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2796474368 bytes 2026-02-20T21:50:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2838107136 bytes 2026-02-20T21:50:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2840827904 bytes 2026-02-20T21:50:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2886864896 bytes 2026-02-20T21:50:42.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2979416064 bytes 2026-02-20T21:50:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3003579392 bytes 2026-02-20T21:50:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3034907648 bytes 2026-02-20T21:50:42.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3067910144 bytes 2026-02-20T21:50:42.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3097513984 bytes 2026-02-20T21:50:42.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3114281984 bytes 2026-02-20T21:50:42.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3150489600 bytes 2026-02-20T21:50:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3151111168 bytes 2026-02-20T21:50:42.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3186874368 bytes 2026-02-20T21:50:42.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3270453248 bytes 2026-02-20T21:50:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3320972288 bytes 2026-02-20T21:50:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3362212864 bytes 2026-02-20T21:50:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3394089984 bytes 2026-02-20T21:50:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3424000000 bytes 2026-02-20T21:50:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3434992640 bytes 2026-02-20T21:50:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3452652544 bytes 2026-02-20T21:50:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3532510208 bytes 2026-02-20T21:50:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3547482112 bytes 2026-02-20T21:50:43.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3558023168 bytes 2026-02-20T21:50:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3649065984 bytes 2026-02-20T21:50:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3720197120 bytes 2026-02-20T21:50:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3737972736 bytes 2026-02-20T21:50:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3758647296 bytes 2026-02-20T21:50:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3795807232 bytes 2026-02-20T21:50:43.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3800009728 bytes 2026-02-20T21:50:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3823702016 bytes 2026-02-20T21:50:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3834285056 bytes 2026-02-20T21:50:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 3856572416 bytes 2026-02-20T21:50:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 3904820224 bytes 2026-02-20T21:50:43.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 3982748672 bytes 2026-02-20T21:50:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 3988947968 bytes 2026-02-20T21:50:43.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4016218112 bytes 2026-02-20T21:50:44.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4030778368 bytes 2026-02-20T21:50:44.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4117542912 bytes 2026-02-20T21:50:44.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4185779200 bytes 2026-02-20T21:50:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4192736256 bytes 2026-02-20T21:50:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4243791872 bytes 2026-02-20T21:50:44.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4365704192 bytes 2026-02-20T21:50:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4397412352 bytes 2026-02-20T21:50:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4422491136 bytes 2026-02-20T21:50:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4488660992 bytes 2026-02-20T21:50:44.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4545092608 bytes 2026-02-20T21:50:44.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4567851008 bytes 2026-02-20T21:50:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4588015616 bytes 2026-02-20T21:50:44.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4669499392 bytes 2026-02-20T21:50:44.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4687811584 bytes 2026-02-20T21:50:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4745878528 bytes 2026-02-20T21:50:44.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4802845696 bytes 2026-02-20T21:50:44.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4885367808 bytes 2026-02-20T21:50:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4968344576 bytes 2026-02-20T21:50:44.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5012439040 bytes 2026-02-20T21:50:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5058113536 bytes 2026-02-20T21:50:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5063206912 bytes 2026-02-20T21:50:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5114142720 bytes 2026-02-20T21:50:45.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5159391232 bytes 2026-02-20T21:50:45.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5223591936 bytes 2026-02-20T21:50:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5229967360 bytes 2026-02-20T21:50:45.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5290166272 bytes 2026-02-20T21:50:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5351535616 bytes 2026-02-20T21:50:45.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5401815040 bytes 2026-02-20T21:50:45.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5490622464 bytes 2026-02-20T21:50:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5549647872 bytes 2026-02-20T21:50:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5599208448 bytes 2026-02-20T21:50:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5628848128 bytes 2026-02-20T21:50:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5649438720 bytes 2026-02-20T21:50:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5703883776 bytes 2026-02-20T21:50:45.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5730203648 bytes 2026-02-20T21:50:45.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5769631744 bytes 2026-02-20T21:50:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5802351616 bytes 2026-02-20T21:50:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5842118656 bytes 2026-02-20T21:50:45.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5879208960 bytes 2026-02-20T21:50:46.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5910611968 bytes 2026-02-20T21:50:46.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5950789632 bytes 2026-02-20T21:50:46.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5985013760 bytes 2026-02-20T21:50:46.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6032534528 bytes 2026-02-20T21:50:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6109971456 bytes 2026-02-20T21:50:46.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6143976448 bytes 2026-02-20T21:50:46.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6168984576 bytes 2026-02-20T21:50:46.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6209218560 bytes 2026-02-20T21:50:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6238170112 bytes 2026-02-20T21:50:46.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6252637184 bytes 2026-02-20T21:50:46.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6270829568 bytes 2026-02-20T21:50:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6305769472 bytes 2026-02-20T21:50:46.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6332393472 bytes 2026-02-20T21:50:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6362881024 bytes 2026-02-20T21:50:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6409874432 bytes 2026-02-20T21:50:46.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6432710656 bytes 2026-02-20T21:50:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6457099264 bytes 2026-02-20T21:50:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6487737344 bytes 2026-02-20T21:50:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6556571648 bytes 2026-02-20T21:50:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6580319232 bytes 2026-02-20T21:50:46.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6613169152 bytes 2026-02-20T21:50:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6626801664 bytes 2026-02-20T21:50:46.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6637837312 bytes 2026-02-20T21:50:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6684767232 bytes 2026-02-20T21:50:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6735544320 bytes 2026-02-20T21:50:47.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6804447232 bytes 2026-02-20T21:50:47.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6852033536 bytes 2026-02-20T21:50:47.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6873492480 bytes 2026-02-20T21:50:47.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 6925077504 bytes 2026-02-20T21:50:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 6938611712 bytes 2026-02-20T21:50:47.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7009054720 bytes 2026-02-20T21:50:47.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7049198592 bytes 2026-02-20T21:50:47.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7130939392 bytes 2026-02-20T21:50:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7142576128 bytes 2026-02-20T21:50:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7208238080 bytes 2026-02-20T21:50:47.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7239815168 bytes 2026-02-20T21:50:47.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7304771584 bytes 2026-02-20T21:50:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7362588672 bytes 2026-02-20T21:50:47.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7390661632 bytes 2026-02-20T21:50:47.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7445824512 bytes 2026-02-20T21:50:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7489626112 bytes 2026-02-20T21:50:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7540568064 bytes 2026-02-20T21:50:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7631407104 bytes 2026-02-20T21:50:48.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7655000064 bytes 2026-02-20T21:50:48.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7709290496 bytes 2026-02-20T21:50:48.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7725625344 bytes 2026-02-20T21:50:48.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7746797568 bytes 2026-02-20T21:50:48.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7796355072 bytes 2026-02-20T21:50:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7885318144 bytes 2026-02-20T21:50:48.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 7909246976 bytes 2026-02-20T21:50:48.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8006438912 bytes 2026-02-20T21:50:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8043705344 bytes 2026-02-20T21:50:48.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8087422976 bytes 2026-02-20T21:50:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8140224512 bytes 2026-02-20T21:50:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8170078208 bytes 2026-02-20T21:50:48.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8219774976 bytes 2026-02-20T21:50:48.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8245074944 bytes 2026-02-20T21:50:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:50:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:50:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:50:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:50:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_18 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 snapshot 128M 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_18 128M 2026-02-20T21:50:48.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:48.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:48.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:48.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:48.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:50:48.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:50:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:50:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:50:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_18 2026-02-20T21:50:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:50:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:50:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_18 snapshot 2026-02-20T21:50:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:50:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_18 snapshot 2026-02-20T21:50:49.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:50:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_18 2026-02-20T21:50:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_18': 2026-02-20T21:50:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:50:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:50:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:50:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13497863e62d 2026-02-20T21:50:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13497863e62d 2026-02-20T21:50:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:50:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:50:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:50:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:50:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:48 2026 2026-02-20T21:50:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:48 2026 2026-02-20T21:50:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:48 2026 2026-02-20T21:50:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:50:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:50:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: a7e538f7-880a-4d2e-9bda-105c14b4adf7 2026-02-20T21:50:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:50:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:50:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_18 2026-02-20T21:50:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:50:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:50:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T21:50:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:50:49.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:50:49.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 86345728 bytes 2026-02-20T21:50:49.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 120038400 bytes 2026-02-20T21:50:49.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 151547904 bytes 2026-02-20T21:50:50.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 200714240 bytes 2026-02-20T21:50:50.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 231249920 bytes 2026-02-20T21:50:50.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 323012608 bytes 2026-02-20T21:50:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 347639808 bytes 2026-02-20T21:50:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 434335744 bytes 2026-02-20T21:50:50.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 527955968 bytes 2026-02-20T21:50:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 561643520 bytes 2026-02-20T21:50:50.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 623127552 bytes 2026-02-20T21:50:50.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 643231744 bytes 2026-02-20T21:50:50.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 708209664 bytes 2026-02-20T21:50:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 755694592 bytes 2026-02-20T21:50:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 858452992 bytes 2026-02-20T21:50:50.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 884766720 bytes 2026-02-20T21:50:50.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 906475520 bytes 2026-02-20T21:50:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 1021449216 bytes 2026-02-20T21:50:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 1069416448 bytes 2026-02-20T21:50:51.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1151681536 bytes 2026-02-20T21:50:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1184482304 bytes 2026-02-20T21:50:51.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1277054976 bytes 2026-02-20T21:50:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1339396096 bytes 2026-02-20T21:50:51.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1411789824 bytes 2026-02-20T21:50:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1439052800 bytes 2026-02-20T21:50:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1501271040 bytes 2026-02-20T21:50:51.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1518962688 bytes 2026-02-20T21:50:51.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1573067776 bytes 2026-02-20T21:50:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1593865216 bytes 2026-02-20T21:50:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1653955584 bytes 2026-02-20T21:50:51.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1704299520 bytes 2026-02-20T21:50:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1781191680 bytes 2026-02-20T21:50:51.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1848988672 bytes 2026-02-20T21:50:51.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1874398208 bytes 2026-02-20T21:50:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1889628160 bytes 2026-02-20T21:50:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1890246656 bytes 2026-02-20T21:50:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1912564736 bytes 2026-02-20T21:50:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1952092160 bytes 2026-02-20T21:50:52.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 2043413504 bytes 2026-02-20T21:50:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 2110185472 bytes 2026-02-20T21:50:52.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 2110776320 bytes 2026-02-20T21:50:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2146662400 bytes 2026-02-20T21:50:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2240293888 bytes 2026-02-20T21:50:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2288782336 bytes 2026-02-20T21:50:52.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2320932864 bytes 2026-02-20T21:50:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2321538048 bytes 2026-02-20T21:50:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2380156928 bytes 2026-02-20T21:50:52.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2430478336 bytes 2026-02-20T21:50:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2431077376 bytes 2026-02-20T21:50:52.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2441035776 bytes 2026-02-20T21:50:52.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2479693824 bytes 2026-02-20T21:50:52.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2516041728 bytes 2026-02-20T21:50:52.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2544041984 bytes 2026-02-20T21:50:52.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2573005824 bytes 2026-02-20T21:50:52.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2598777856 bytes 2026-02-20T21:50:53.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2643461120 bytes 2026-02-20T21:50:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2696808448 bytes 2026-02-20T21:50:53.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2749906944 bytes 2026-02-20T21:50:53.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2816048128 bytes 2026-02-20T21:50:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2871950336 bytes 2026-02-20T21:50:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2930921472 bytes 2026-02-20T21:50:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2946297856 bytes 2026-02-20T21:50:53.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2987412480 bytes 2026-02-20T21:50:53.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3021532160 bytes 2026-02-20T21:50:53.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3031479296 bytes 2026-02-20T21:50:53.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3058617344 bytes 2026-02-20T21:50:53.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3074025472 bytes 2026-02-20T21:50:54.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3135784960 bytes 2026-02-20T21:50:54.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3196444672 bytes 2026-02-20T21:50:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3228518400 bytes 2026-02-20T21:50:54.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3256629248 bytes 2026-02-20T21:50:54.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3275328512 bytes 2026-02-20T21:50:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3300259840 bytes 2026-02-20T21:50:54.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3317108736 bytes 2026-02-20T21:50:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3331551232 bytes 2026-02-20T21:50:54.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3338018816 bytes 2026-02-20T21:50:54.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3395509248 bytes 2026-02-20T21:50:54.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3409830912 bytes 2026-02-20T21:50:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3454256128 bytes 2026-02-20T21:50:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3487660032 bytes 2026-02-20T21:50:54.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3519393792 bytes 2026-02-20T21:50:54.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3560086528 bytes 2026-02-20T21:50:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3567250432 bytes 2026-02-20T21:50:54.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3596341248 bytes 2026-02-20T21:50:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3607431168 bytes 2026-02-20T21:50:54.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3618665472 bytes 2026-02-20T21:50:54.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3670929408 bytes 2026-02-20T21:50:54.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3750471680 bytes 2026-02-20T21:50:54.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3753579520 bytes 2026-02-20T21:50:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3758568448 bytes 2026-02-20T21:50:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3851944960 bytes 2026-02-20T21:50:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3882967040 bytes 2026-02-20T21:50:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3907640320 bytes 2026-02-20T21:50:55.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3937405952 bytes 2026-02-20T21:50:55.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3996761088 bytes 2026-02-20T21:50:55.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4088328192 bytes 2026-02-20T21:50:55.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4108985344 bytes 2026-02-20T21:50:55.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4124033024 bytes 2026-02-20T21:50:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4176211968 bytes 2026-02-20T21:50:55.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4193589248 bytes 2026-02-20T21:50:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4238174208 bytes 2026-02-20T21:50:55.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4241910784 bytes 2026-02-20T21:50:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4300931072 bytes 2026-02-20T21:50:55.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4383568896 bytes 2026-02-20T21:50:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4410395648 bytes 2026-02-20T21:50:55.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4511185920 bytes 2026-02-20T21:50:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4536044544 bytes 2026-02-20T21:50:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4536683520 bytes 2026-02-20T21:50:55.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4561064960 bytes 2026-02-20T21:50:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4592352256 bytes 2026-02-20T21:50:56.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4631351296 bytes 2026-02-20T21:50:56.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4685009920 bytes 2026-02-20T21:50:56.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4748489728 bytes 2026-02-20T21:50:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4802471936 bytes 2026-02-20T21:50:56.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4838952960 bytes 2026-02-20T21:50:56.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4884274176 bytes 2026-02-20T21:50:56.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4898979840 bytes 2026-02-20T21:50:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4958920704 bytes 2026-02-20T21:50:56.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5006300160 bytes 2026-02-20T21:50:56.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5049584640 bytes 2026-02-20T21:50:56.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5105191936 bytes 2026-02-20T21:50:56.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5140439040 bytes 2026-02-20T21:50:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5191979008 bytes 2026-02-20T21:50:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5238325248 bytes 2026-02-20T21:50:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5318640640 bytes 2026-02-20T21:50:56.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5349943296 bytes 2026-02-20T21:50:56.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5421287424 bytes 2026-02-20T21:50:56.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5475663872 bytes 2026-02-20T21:50:56.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5532190720 bytes 2026-02-20T21:50:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5539919872 bytes 2026-02-20T21:50:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5561216000 bytes 2026-02-20T21:50:57.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5612967936 bytes 2026-02-20T21:50:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5637524480 bytes 2026-02-20T21:50:57.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5644896256 bytes 2026-02-20T21:50:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5677123584 bytes 2026-02-20T21:50:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5741483008 bytes 2026-02-20T21:50:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5792830464 bytes 2026-02-20T21:50:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5841838080 bytes 2026-02-20T21:50:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5896530944 bytes 2026-02-20T21:50:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5952913408 bytes 2026-02-20T21:50:57.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5991533568 bytes 2026-02-20T21:50:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6001178624 bytes 2026-02-20T21:50:57.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6064711680 bytes 2026-02-20T21:50:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6094930944 bytes 2026-02-20T21:50:57.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6123257856 bytes 2026-02-20T21:50:57.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6183729152 bytes 2026-02-20T21:50:57.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6196468736 bytes 2026-02-20T21:50:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6217482240 bytes 2026-02-20T21:50:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6279010304 bytes 2026-02-20T21:50:57.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6363008000 bytes 2026-02-20T21:50:57.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6420188160 bytes 2026-02-20T21:50:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6442155008 bytes 2026-02-20T21:50:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6461822976 bytes 2026-02-20T21:50:58.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6505566208 bytes 2026-02-20T21:50:58.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6544360448 bytes 2026-02-20T21:50:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6578877440 bytes 2026-02-20T21:50:58.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6665901056 bytes 2026-02-20T21:50:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6713446400 bytes 2026-02-20T21:50:58.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6742347776 bytes 2026-02-20T21:50:58.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6752358400 bytes 2026-02-20T21:50:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6777451520 bytes 2026-02-20T21:50:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6800044032 bytes 2026-02-20T21:50:58.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6800663552 bytes 2026-02-20T21:50:58.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6827612160 bytes 2026-02-20T21:50:58.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6874320896 bytes 2026-02-20T21:50:58.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6943752192 bytes 2026-02-20T21:50:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6982422528 bytes 2026-02-20T21:50:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7030120448 bytes 2026-02-20T21:50:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:50:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:50:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:50:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:50:58.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_19 2026-02-20T21:50:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 snapshot 128M 2026-02-20T21:50:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:50:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:58.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_19 128M 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:50:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:50:58.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_19 2026-02-20T21:50:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:50:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:50:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:50:58.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_19 snapshot 2026-02-20T21:50:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:50:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:50:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_19 snapshot 2026-02-20T21:50:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:50:59.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_19 2026-02-20T21:50:59.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_19': 2026-02-20T21:50:59.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 1355a6f6994 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.1355a6f6994 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:58 2026 2026-02-20T21:50:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:58 2026 2026-02-20T21:50:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:58 2026 2026-02-20T21:50:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:50:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:50:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 7494457f-5c66-44ae-b871-096697c31154 2026-02-20T21:50:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:50:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:50:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_19 2026-02-20T21:50:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:50:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:50:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:50:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:50:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=7 2026-02-20T21:50:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:50:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:50:59.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 36746240 bytes 2026-02-20T21:50:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 70157312 bytes 2026-02-20T21:50:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 121279488 bytes 2026-02-20T21:51:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 172199936 bytes 2026-02-20T21:51:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 218261504 bytes 2026-02-20T21:51:00.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 277208064 bytes 2026-02-20T21:51:00.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 332801024 bytes 2026-02-20T21:51:00.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 367000576 bytes 2026-02-20T21:51:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 378253312 bytes 2026-02-20T21:51:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 426995712 bytes 2026-02-20T21:51:00.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 448873472 bytes 2026-02-20T21:51:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 467348480 bytes 2026-02-20T21:51:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 473202688 bytes 2026-02-20T21:51:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 542704640 bytes 2026-02-20T21:51:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 597884928 bytes 2026-02-20T21:51:00.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 618130432 bytes 2026-02-20T21:51:00.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 651261952 bytes 2026-02-20T21:51:00.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 656413696 bytes 2026-02-20T21:51:00.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 707414016 bytes 2026-02-20T21:51:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 804530176 bytes 2026-02-20T21:51:00.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 847298560 bytes 2026-02-20T21:51:00.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 906466304 bytes 2026-02-20T21:51:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 907067392 bytes 2026-02-20T21:51:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 934654976 bytes 2026-02-20T21:51:00.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 961615872 bytes 2026-02-20T21:51:01.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1014721536 bytes 2026-02-20T21:51:01.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1091921920 bytes 2026-02-20T21:51:01.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1123881984 bytes 2026-02-20T21:51:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1205966848 bytes 2026-02-20T21:51:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1238370304 bytes 2026-02-20T21:51:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1284334592 bytes 2026-02-20T21:51:01.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1327860736 bytes 2026-02-20T21:51:01.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1372814336 bytes 2026-02-20T21:51:01.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1404561408 bytes 2026-02-20T21:51:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1445998592 bytes 2026-02-20T21:51:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1486241792 bytes 2026-02-20T21:51:01.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1579924480 bytes 2026-02-20T21:51:01.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1665426432 bytes 2026-02-20T21:51:01.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1738140672 bytes 2026-02-20T21:51:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1782759424 bytes 2026-02-20T21:51:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1805037568 bytes 2026-02-20T21:51:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1877036032 bytes 2026-02-20T21:51:01.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1906649088 bytes 2026-02-20T21:51:01.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1933765632 bytes 2026-02-20T21:51:02.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1986085888 bytes 2026-02-20T21:51:02.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2077871104 bytes 2026-02-20T21:51:02.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2115000320 bytes 2026-02-20T21:51:02.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2163604480 bytes 2026-02-20T21:51:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2176468992 bytes 2026-02-20T21:51:02.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2216311808 bytes 2026-02-20T21:51:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2216918016 bytes 2026-02-20T21:51:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2274461696 bytes 2026-02-20T21:51:02.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2298104832 bytes 2026-02-20T21:51:02.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2352904192 bytes 2026-02-20T21:51:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2358216704 bytes 2026-02-20T21:51:02.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2433703936 bytes 2026-02-20T21:51:02.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2519021568 bytes 2026-02-20T21:51:02.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2553985024 bytes 2026-02-20T21:51:02.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2594042880 bytes 2026-02-20T21:51:02.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2618679296 bytes 2026-02-20T21:51:02.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2643598336 bytes 2026-02-20T21:51:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2671531008 bytes 2026-02-20T21:51:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2695506944 bytes 2026-02-20T21:51:02.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2702466048 bytes 2026-02-20T21:51:03.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2800984064 bytes 2026-02-20T21:51:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2861389824 bytes 2026-02-20T21:51:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2871415808 bytes 2026-02-20T21:51:03.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2902662144 bytes 2026-02-20T21:51:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3021386752 bytes 2026-02-20T21:51:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3044816896 bytes 2026-02-20T21:51:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3129814016 bytes 2026-02-20T21:51:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3189284864 bytes 2026-02-20T21:51:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3189903360 bytes 2026-02-20T21:51:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3232931840 bytes 2026-02-20T21:51:03.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3273698304 bytes 2026-02-20T21:51:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3300427776 bytes 2026-02-20T21:51:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3342217216 bytes 2026-02-20T21:51:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3368531968 bytes 2026-02-20T21:51:04.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3445238784 bytes 2026-02-20T21:51:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3542102016 bytes 2026-02-20T21:51:04.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3595987968 bytes 2026-02-20T21:51:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3617003520 bytes 2026-02-20T21:51:04.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3659517952 bytes 2026-02-20T21:51:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3676153856 bytes 2026-02-20T21:51:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3743747072 bytes 2026-02-20T21:51:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3822953472 bytes 2026-02-20T21:51:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3856247808 bytes 2026-02-20T21:51:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3886357504 bytes 2026-02-20T21:51:04.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3911837696 bytes 2026-02-20T21:51:04.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3953920000 bytes 2026-02-20T21:51:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4009592832 bytes 2026-02-20T21:51:04.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4046658560 bytes 2026-02-20T21:51:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4058949632 bytes 2026-02-20T21:51:04.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4149174272 bytes 2026-02-20T21:51:04.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4174683136 bytes 2026-02-20T21:51:04.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4201698304 bytes 2026-02-20T21:51:04.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4234707968 bytes 2026-02-20T21:51:04.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4248896512 bytes 2026-02-20T21:51:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4289814528 bytes 2026-02-20T21:51:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4300191744 bytes 2026-02-20T21:51:05.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4335091712 bytes 2026-02-20T21:51:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4364753920 bytes 2026-02-20T21:51:05.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4389772288 bytes 2026-02-20T21:51:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4449592320 bytes 2026-02-20T21:51:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4484088832 bytes 2026-02-20T21:51:05.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4522158080 bytes 2026-02-20T21:51:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4549987328 bytes 2026-02-20T21:51:05.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4650771456 bytes 2026-02-20T21:51:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4719210496 bytes 2026-02-20T21:51:05.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4790275072 bytes 2026-02-20T21:51:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4810664960 bytes 2026-02-20T21:51:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4890744832 bytes 2026-02-20T21:51:06.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4933060608 bytes 2026-02-20T21:51:06.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5035345920 bytes 2026-02-20T21:51:06.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5050407936 bytes 2026-02-20T21:51:06.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5115290624 bytes 2026-02-20T21:51:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5167387648 bytes 2026-02-20T21:51:06.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5208489984 bytes 2026-02-20T21:51:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5265992704 bytes 2026-02-20T21:51:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5312008192 bytes 2026-02-20T21:51:06.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5316125696 bytes 2026-02-20T21:51:06.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5405487104 bytes 2026-02-20T21:51:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5415168000 bytes 2026-02-20T21:51:06.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5475191808 bytes 2026-02-20T21:51:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5535679488 bytes 2026-02-20T21:51:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:51:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:51:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:51:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_20 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 snapshot 128M 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_20 128M 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_20 2026-02-20T21:51:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:51:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:51:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_20 snapshot 2026-02-20T21:51:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:51:06.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:06.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_20 snapshot 2026-02-20T21:51:07.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:51:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_20 2026-02-20T21:51:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_20': 2026-02-20T21:51:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:51:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:51:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13617c9f591f 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13617c9f591f 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:06 2026 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:06 2026 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:06 2026 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:51:07.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:51:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 919065ea-03af-418d-938d-c4f0d0f3196f 2026-02-20T21:51:07.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:51:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:51:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_20 2026-02-20T21:51:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:51:07.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:51:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=12 2026-02-20T21:51:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:51:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:51:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 31000576 bytes 2026-02-20T21:51:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 54171648 bytes 2026-02-20T21:51:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 69770240 bytes 2026-02-20T21:51:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 70398976 bytes 2026-02-20T21:51:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 103510016 bytes 2026-02-20T21:51:08.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 113947648 bytes 2026-02-20T21:51:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 156158976 bytes 2026-02-20T21:51:08.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 263696384 bytes 2026-02-20T21:51:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 307277824 bytes 2026-02-20T21:51:08.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 354578432 bytes 2026-02-20T21:51:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 410902528 bytes 2026-02-20T21:51:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 420587520 bytes 2026-02-20T21:51:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 496384000 bytes 2026-02-20T21:51:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 535571456 bytes 2026-02-20T21:51:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 725360640 bytes 2026-02-20T21:51:08.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 812223488 bytes 2026-02-20T21:51:08.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 814520320 bytes 2026-02-20T21:51:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 815777792 bytes 2026-02-20T21:51:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 857366528 bytes 2026-02-20T21:51:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 908427264 bytes 2026-02-20T21:51:08.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 942337024 bytes 2026-02-20T21:51:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1021378560 bytes 2026-02-20T21:51:08.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1060985856 bytes 2026-02-20T21:51:08.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1100060672 bytes 2026-02-20T21:51:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1179131904 bytes 2026-02-20T21:51:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1191336960 bytes 2026-02-20T21:51:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1249790976 bytes 2026-02-20T21:51:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1293732864 bytes 2026-02-20T21:51:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1422606336 bytes 2026-02-20T21:51:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1432444928 bytes 2026-02-20T21:51:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1451662336 bytes 2026-02-20T21:51:09.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1531707392 bytes 2026-02-20T21:51:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1590498304 bytes 2026-02-20T21:51:09.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1613024256 bytes 2026-02-20T21:51:09.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1708048384 bytes 2026-02-20T21:51:09.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1787352064 bytes 2026-02-20T21:51:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1848853504 bytes 2026-02-20T21:51:09.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1920921600 bytes 2026-02-20T21:51:09.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1967981568 bytes 2026-02-20T21:51:09.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1968594944 bytes 2026-02-20T21:51:09.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 2031210496 bytes 2026-02-20T21:51:09.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2128983040 bytes 2026-02-20T21:51:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2240685056 bytes 2026-02-20T21:51:10.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2256594944 bytes 2026-02-20T21:51:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2298999808 bytes 2026-02-20T21:51:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2362147840 bytes 2026-02-20T21:51:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2363601920 bytes 2026-02-20T21:51:10.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2382001152 bytes 2026-02-20T21:51:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2428334080 bytes 2026-02-20T21:51:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2490942464 bytes 2026-02-20T21:51:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2545959936 bytes 2026-02-20T21:51:10.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2554960896 bytes 2026-02-20T21:51:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2652477440 bytes 2026-02-20T21:51:10.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2728677376 bytes 2026-02-20T21:51:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2767424512 bytes 2026-02-20T21:51:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2822056960 bytes 2026-02-20T21:51:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2845504512 bytes 2026-02-20T21:51:10.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2900384768 bytes 2026-02-20T21:51:10.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2927081472 bytes 2026-02-20T21:51:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2948850688 bytes 2026-02-20T21:51:10.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 3018432512 bytes 2026-02-20T21:51:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 3067677696 bytes 2026-02-20T21:51:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 3106358272 bytes 2026-02-20T21:51:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3130598400 bytes 2026-02-20T21:51:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3165925376 bytes 2026-02-20T21:51:11.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3196503040 bytes 2026-02-20T21:51:11.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3225456640 bytes 2026-02-20T21:51:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3302397952 bytes 2026-02-20T21:51:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3363388416 bytes 2026-02-20T21:51:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3372492800 bytes 2026-02-20T21:51:11.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3412121600 bytes 2026-02-20T21:51:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3505937408 bytes 2026-02-20T21:51:11.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3537396736 bytes 2026-02-20T21:51:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3564116992 bytes 2026-02-20T21:51:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3711301632 bytes 2026-02-20T21:51:11.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3751034880 bytes 2026-02-20T21:51:11.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3768142848 bytes 2026-02-20T21:51:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3828471808 bytes 2026-02-20T21:51:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3903882240 bytes 2026-02-20T21:51:11.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3915556864 bytes 2026-02-20T21:51:11.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3937944576 bytes 2026-02-20T21:51:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3988628480 bytes 2026-02-20T21:51:11.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 4004025344 bytes 2026-02-20T21:51:11.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 4029628416 bytes 2026-02-20T21:51:12.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 4039835648 bytes 2026-02-20T21:51:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 4058195968 bytes 2026-02-20T21:51:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 4092043264 bytes 2026-02-20T21:51:12.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4125302784 bytes 2026-02-20T21:51:12.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4192291840 bytes 2026-02-20T21:51:12.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4241094656 bytes 2026-02-20T21:51:12.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4283670528 bytes 2026-02-20T21:51:12.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4352020480 bytes 2026-02-20T21:51:12.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4406125568 bytes 2026-02-20T21:51:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4411047936 bytes 2026-02-20T21:51:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4485797888 bytes 2026-02-20T21:51:12.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4540247040 bytes 2026-02-20T21:51:12.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4577985536 bytes 2026-02-20T21:51:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4604608512 bytes 2026-02-20T21:51:12.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4630855680 bytes 2026-02-20T21:51:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4658092032 bytes 2026-02-20T21:51:12.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4694501376 bytes 2026-02-20T21:51:12.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4753426432 bytes 2026-02-20T21:51:12.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4856288256 bytes 2026-02-20T21:51:12.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4900810752 bytes 2026-02-20T21:51:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4954340352 bytes 2026-02-20T21:51:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 5003012096 bytes 2026-02-20T21:51:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 5075241984 bytes 2026-02-20T21:51:13.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 5127628800 bytes 2026-02-20T21:51:13.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 5151344640 bytes 2026-02-20T21:51:13.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5187946496 bytes 2026-02-20T21:51:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5251709952 bytes 2026-02-20T21:51:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5280443392 bytes 2026-02-20T21:51:13.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5347296256 bytes 2026-02-20T21:51:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5358198784 bytes 2026-02-20T21:51:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5384244224 bytes 2026-02-20T21:51:13.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5389787136 bytes 2026-02-20T21:51:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5409624064 bytes 2026-02-20T21:51:13.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5491637248 bytes 2026-02-20T21:51:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5540575232 bytes 2026-02-20T21:51:13.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5587645440 bytes 2026-02-20T21:51:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5641162752 bytes 2026-02-20T21:51:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5689621504 bytes 2026-02-20T21:51:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5733176320 bytes 2026-02-20T21:51:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5831368704 bytes 2026-02-20T21:51:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5883086848 bytes 2026-02-20T21:51:14.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5916400640 bytes 2026-02-20T21:51:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5971787776 bytes 2026-02-20T21:51:14.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 6029208576 bytes 2026-02-20T21:51:14.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 6080465920 bytes 2026-02-20T21:51:14.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 6088843264 bytes 2026-02-20T21:51:14.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 6106411008 bytes 2026-02-20T21:51:14.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 6174411776 bytes 2026-02-20T21:51:14.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 6227016704 bytes 2026-02-20T21:51:14.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 6242266112 bytes 2026-02-20T21:51:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 6266402816 bytes 2026-02-20T21:51:14.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 6329293824 bytes 2026-02-20T21:51:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 6366167040 bytes 2026-02-20T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6480296960 bytes 2026-02-20T21:51:14.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6542197760 bytes 2026-02-20T21:51:14.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6578767872 bytes 2026-02-20T21:51:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6614848512 bytes 2026-02-20T21:51:14.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6657856512 bytes 2026-02-20T21:51:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6708005888 bytes 2026-02-20T21:51:14.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6722565120 bytes 2026-02-20T21:51:14.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6744012800 bytes 2026-02-20T21:51:14.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6744634368 bytes 2026-02-20T21:51:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6785506304 bytes 2026-02-20T21:51:15.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6838534144 bytes 2026-02-20T21:51:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6907182080 bytes 2026-02-20T21:51:15.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6975564800 bytes 2026-02-20T21:51:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6976157696 bytes 2026-02-20T21:51:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 7021581312 bytes 2026-02-20T21:51:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 7035106304 bytes 2026-02-20T21:51:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 7053415424 bytes 2026-02-20T21:51:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 7076578304 bytes 2026-02-20T21:51:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 7117330432 bytes 2026-02-20T21:51:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 7129368576 bytes 2026-02-20T21:51:15.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 7172371456 bytes 2026-02-20T21:51:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 7287561216 bytes 2026-02-20T21:51:15.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 7301893120 bytes 2026-02-20T21:51:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7386905600 bytes 2026-02-20T21:51:15.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7400219648 bytes 2026-02-20T21:51:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7425097728 bytes 2026-02-20T21:51:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7453807616 bytes 2026-02-20T21:51:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7491589120 bytes 2026-02-20T21:51:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7533565952 bytes 2026-02-20T21:51:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7603206144 bytes 2026-02-20T21:51:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7645194240 bytes 2026-02-20T21:51:16.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7685726208 bytes 2026-02-20T21:51:16.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7762327552 bytes 2026-02-20T21:51:16.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7824836608 bytes 2026-02-20T21:51:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7865607168 bytes 2026-02-20T21:51:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7893666816 bytes 2026-02-20T21:51:16.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7940645888 bytes 2026-02-20T21:51:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7976782848 bytes 2026-02-20T21:51:16.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 8002412544 bytes 2026-02-20T21:51:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 8041659392 bytes 2026-02-20T21:51:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 8104502272 bytes 2026-02-20T21:51:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 8184997888 bytes 2026-02-20T21:51:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 8190289920 bytes 2026-02-20T21:51:16.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 8282200064 bytes 2026-02-20T21:51:16.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 8290816000 bytes 2026-02-20T21:51:16.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 8296112128 bytes 2026-02-20T21:51:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 8333243392 bytes 2026-02-20T21:51:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 8359301120 bytes 2026-02-20T21:51:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 8408819712 bytes 2026-02-20T21:51:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8458985472 bytes 2026-02-20T21:51:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8508810240 bytes 2026-02-20T21:51:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8546759680 bytes 2026-02-20T21:51:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8573451264 bytes 2026-02-20T21:51:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8581118976 bytes 2026-02-20T21:51:17.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8615833600 bytes 2026-02-20T21:51:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8673811456 bytes 2026-02-20T21:51:17.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8724043776 bytes 2026-02-20T21:51:17.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8736690176 bytes 2026-02-20T21:51:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8854075392 bytes 2026-02-20T21:51:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8905420800 bytes 2026-02-20T21:51:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8906013696 bytes 2026-02-20T21:51:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8921863168 bytes 2026-02-20T21:51:17.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8982230016 bytes 2026-02-20T21:51:17.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 9056326656 bytes 2026-02-20T21:51:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 9056954368 bytes 2026-02-20T21:51:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 9075290112 bytes 2026-02-20T21:51:17.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 9129525248 bytes 2026-02-20T21:51:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 9180328960 bytes 2026-02-20T21:51:17.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 9233081344 bytes 2026-02-20T21:51:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 9255561216 bytes 2026-02-20T21:51:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 9293308928 bytes 2026-02-20T21:51:17.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 9340889088 bytes 2026-02-20T21:51:17.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9415002112 bytes 2026-02-20T21:51:18.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9476826112 bytes 2026-02-20T21:51:18.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9525399552 bytes 2026-02-20T21:51:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9548185600 bytes 2026-02-20T21:51:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9628792832 bytes 2026-02-20T21:51:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9691833344 bytes 2026-02-20T21:51:18.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9692401664 bytes 2026-02-20T21:51:18.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9717175296 bytes 2026-02-20T21:51:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9795278848 bytes 2026-02-20T21:51:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9822186496 bytes 2026-02-20T21:51:18.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9886821376 bytes 2026-02-20T21:51:18.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9917254656 bytes 2026-02-20T21:51:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9957676032 bytes 2026-02-20T21:51:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9994415104 bytes 2026-02-20T21:51:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 10058277888 bytes 2026-02-20T21:51:18.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 10086824960 bytes 2026-02-20T21:51:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 10126151680 bytes 2026-02-20T21:51:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 10147438592 bytes 2026-02-20T21:51:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 10190479360 bytes 2026-02-20T21:51:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 10241076224 bytes 2026-02-20T21:51:19.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 10270159872 bytes 2026-02-20T21:51:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 10328745984 bytes 2026-02-20T21:51:19.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 10338593792 bytes 2026-02-20T21:51:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 10396440576 bytes 2026-02-20T21:51:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 10462046208 bytes 2026-02-20T21:51:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 10490705920 bytes 2026-02-20T21:51:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 10546070528 bytes 2026-02-20T21:51:19.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 10591552512 bytes 2026-02-20T21:51:19.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 10646425600 bytes 2026-02-20T21:51:19.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 10708431872 bytes 2026-02-20T21:51:19.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 10720261120 bytes 2026-02-20T21:51:19.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 10762436608 bytes 2026-02-20T21:51:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10830915584 bytes 2026-02-20T21:51:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:51:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:51:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:51:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_21 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 snapshot 128M 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_21 128M 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_21 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_21 snapshot 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:19.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_21 snapshot 2026-02-20T21:51:20.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:51:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_21 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_21': 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 136d8222d6bd 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.136d8222d6bd 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:19 2026 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:19 2026 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:19 2026 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: ea30739c-7e9c-4234-88fe-ea59f7828681 2026-02-20T21:51:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:51:20.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:51:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_21 2026-02-20T21:51:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:51:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:51:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T21:51:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:51:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:51:20.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 28603392 bytes 2026-02-20T21:51:20.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 67976192 bytes 2026-02-20T21:51:20.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 106437632 bytes 2026-02-20T21:51:21.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 153467904 bytes 2026-02-20T21:51:21.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 224287744 bytes 2026-02-20T21:51:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 262889472 bytes 2026-02-20T21:51:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 325367808 bytes 2026-02-20T21:51:21.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 356465664 bytes 2026-02-20T21:51:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 419935232 bytes 2026-02-20T21:51:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 494390272 bytes 2026-02-20T21:51:21.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 558239744 bytes 2026-02-20T21:51:21.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 573304832 bytes 2026-02-20T21:51:21.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 630946816 bytes 2026-02-20T21:51:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 666234880 bytes 2026-02-20T21:51:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 673228800 bytes 2026-02-20T21:51:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 692336640 bytes 2026-02-20T21:51:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 766477312 bytes 2026-02-20T21:51:21.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 767083520 bytes 2026-02-20T21:51:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 835574784 bytes 2026-02-20T21:51:21.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 883607552 bytes 2026-02-20T21:51:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 981039104 bytes 2026-02-20T21:51:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1032667136 bytes 2026-02-20T21:51:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1059886080 bytes 2026-02-20T21:51:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1096750080 bytes 2026-02-20T21:51:22.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1097385984 bytes 2026-02-20T21:51:22.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1156792320 bytes 2026-02-20T21:51:22.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1158061056 bytes 2026-02-20T21:51:22.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1192538112 bytes 2026-02-20T21:51:22.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1240237056 bytes 2026-02-20T21:51:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1273643008 bytes 2026-02-20T21:51:22.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1304389632 bytes 2026-02-20T21:51:22.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1305026560 bytes 2026-02-20T21:51:22.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1386115072 bytes 2026-02-20T21:51:22.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1399901184 bytes 2026-02-20T21:51:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1505063936 bytes 2026-02-20T21:51:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1544014848 bytes 2026-02-20T21:51:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1595555840 bytes 2026-02-20T21:51:22.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1629206528 bytes 2026-02-20T21:51:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1664783360 bytes 2026-02-20T21:51:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1697728512 bytes 2026-02-20T21:51:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1818395648 bytes 2026-02-20T21:51:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1863492608 bytes 2026-02-20T21:51:23.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1901325312 bytes 2026-02-20T21:51:23.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1969589248 bytes 2026-02-20T21:51:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2002271232 bytes 2026-02-20T21:51:23.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2133743616 bytes 2026-02-20T21:51:23.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2134347776 bytes 2026-02-20T21:51:23.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2168314880 bytes 2026-02-20T21:51:23.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2182941696 bytes 2026-02-20T21:51:23.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2204701696 bytes 2026-02-20T21:51:23.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2226181120 bytes 2026-02-20T21:51:23.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2247332864 bytes 2026-02-20T21:51:23.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2271968256 bytes 2026-02-20T21:51:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2295989248 bytes 2026-02-20T21:51:23.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2334183424 bytes 2026-02-20T21:51:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2369476608 bytes 2026-02-20T21:51:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2370315264 bytes 2026-02-20T21:51:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2420124672 bytes 2026-02-20T21:51:23.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2473412608 bytes 2026-02-20T21:51:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2506357760 bytes 2026-02-20T21:51:23.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2540851200 bytes 2026-02-20T21:51:23.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2615017472 bytes 2026-02-20T21:51:24.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2688122880 bytes 2026-02-20T21:51:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2730946560 bytes 2026-02-20T21:51:24.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2809234432 bytes 2026-02-20T21:51:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2910269440 bytes 2026-02-20T21:51:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2954735616 bytes 2026-02-20T21:51:24.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2965655552 bytes 2026-02-20T21:51:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2980016128 bytes 2026-02-20T21:51:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3016552448 bytes 2026-02-20T21:51:24.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3059235840 bytes 2026-02-20T21:51:24.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3081098240 bytes 2026-02-20T21:51:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3108569088 bytes 2026-02-20T21:51:24.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3141203968 bytes 2026-02-20T21:51:24.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3157256192 bytes 2026-02-20T21:51:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3248543744 bytes 2026-02-20T21:51:24.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3249162240 bytes 2026-02-20T21:51:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3263801344 bytes 2026-02-20T21:51:24.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3271453696 bytes 2026-02-20T21:51:24.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3333400576 bytes 2026-02-20T21:51:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3411893248 bytes 2026-02-20T21:51:24.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3451500544 bytes 2026-02-20T21:51:25.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3533545472 bytes 2026-02-20T21:51:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3564864512 bytes 2026-02-20T21:51:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3644496896 bytes 2026-02-20T21:51:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3688836096 bytes 2026-02-20T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3728514048 bytes 2026-02-20T21:51:25.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3838546944 bytes 2026-02-20T21:51:25.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3850196992 bytes 2026-02-20T21:51:25.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3942659072 bytes 2026-02-20T21:51:25.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4041710592 bytes 2026-02-20T21:51:25.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4098872320 bytes 2026-02-20T21:51:25.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4155908096 bytes 2026-02-20T21:51:25.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4175607808 bytes 2026-02-20T21:51:25.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4176221184 bytes 2026-02-20T21:51:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4198574080 bytes 2026-02-20T21:51:25.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4233969664 bytes 2026-02-20T21:51:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4234569728 bytes 2026-02-20T21:51:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4291322880 bytes 2026-02-20T21:51:25.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4321854464 bytes 2026-02-20T21:51:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4347535360 bytes 2026-02-20T21:51:26.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4370043904 bytes 2026-02-20T21:51:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4389551104 bytes 2026-02-20T21:51:26.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4412610560 bytes 2026-02-20T21:51:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4424744960 bytes 2026-02-20T21:51:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4442989568 bytes 2026-02-20T21:51:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4460136448 bytes 2026-02-20T21:51:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4543823872 bytes 2026-02-20T21:51:26.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4552185856 bytes 2026-02-20T21:51:26.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4569641984 bytes 2026-02-20T21:51:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4605685760 bytes 2026-02-20T21:51:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4619998208 bytes 2026-02-20T21:51:26.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4696437760 bytes 2026-02-20T21:51:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4802067456 bytes 2026-02-20T21:51:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4823879680 bytes 2026-02-20T21:51:26.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4838522880 bytes 2026-02-20T21:51:26.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4848223232 bytes 2026-02-20T21:51:26.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4898624512 bytes 2026-02-20T21:51:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4936858624 bytes 2026-02-20T21:51:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4979420160 bytes 2026-02-20T21:51:26.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4998270976 bytes 2026-02-20T21:51:27.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5025714176 bytes 2026-02-20T21:51:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5059168256 bytes 2026-02-20T21:51:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5104760832 bytes 2026-02-20T21:51:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5129042944 bytes 2026-02-20T21:51:27.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5168318464 bytes 2026-02-20T21:51:27.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5195564032 bytes 2026-02-20T21:51:27.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5234159616 bytes 2026-02-20T21:51:27.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5315971072 bytes 2026-02-20T21:51:27.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5325087744 bytes 2026-02-20T21:51:27.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5355430912 bytes 2026-02-20T21:51:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5390859264 bytes 2026-02-20T21:51:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5431265280 bytes 2026-02-20T21:51:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5501115392 bytes 2026-02-20T21:51:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5560415232 bytes 2026-02-20T21:51:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5609602048 bytes 2026-02-20T21:51:27.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5657261056 bytes 2026-02-20T21:51:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5677341696 bytes 2026-02-20T21:51:27.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5691620352 bytes 2026-02-20T21:51:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5721693184 bytes 2026-02-20T21:51:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5781413888 bytes 2026-02-20T21:51:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5803776000 bytes 2026-02-20T21:51:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5844937728 bytes 2026-02-20T21:51:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5883187200 bytes 2026-02-20T21:51:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5883802624 bytes 2026-02-20T21:51:28.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5910622208 bytes 2026-02-20T21:51:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6007739392 bytes 2026-02-20T21:51:28.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6049787904 bytes 2026-02-20T21:51:28.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6130038784 bytes 2026-02-20T21:51:28.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6145700864 bytes 2026-02-20T21:51:28.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6192762880 bytes 2026-02-20T21:51:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6214847488 bytes 2026-02-20T21:51:28.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6226608128 bytes 2026-02-20T21:51:29.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6249663488 bytes 2026-02-20T21:51:29.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6289787904 bytes 2026-02-20T21:51:29.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6350299136 bytes 2026-02-20T21:51:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6440468480 bytes 2026-02-20T21:51:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6519993344 bytes 2026-02-20T21:51:29.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6573044736 bytes 2026-02-20T21:51:29.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6585575424 bytes 2026-02-20T21:51:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6636472320 bytes 2026-02-20T21:51:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6639912960 bytes 2026-02-20T21:51:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6675345408 bytes 2026-02-20T21:51:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6713137152 bytes 2026-02-20T21:51:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:51:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:51:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:51:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_22 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 snapshot 128M 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_22 128M 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:29.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_22 2026-02-20T21:51:29.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:51:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:51:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_22 snapshot 2026-02-20T21:51:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:51:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_22 snapshot 2026-02-20T21:51:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:51:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_22 2026-02-20T21:51:30.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_22': 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13795bbf2af4 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13795bbf2af4 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:29 2026 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:29 2026 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:29 2026 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 34a71a4e-36fa-455c-a493-c55de2c7e753 2026-02-20T21:51:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:51:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:51:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_22 2026-02-20T21:51:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:51:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:51:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=11 2026-02-20T21:51:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:51:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:51:30.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 19687424 bytes 2026-02-20T21:51:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 44012544 bytes 2026-02-20T21:51:30.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 116961280 bytes 2026-02-20T21:51:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 181364736 bytes 2026-02-20T21:51:30.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 275500032 bytes 2026-02-20T21:51:30.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 363113472 bytes 2026-02-20T21:51:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 388255744 bytes 2026-02-20T21:51:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 439749632 bytes 2026-02-20T21:51:31.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 511147008 bytes 2026-02-20T21:51:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 563964928 bytes 2026-02-20T21:51:31.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 612192256 bytes 2026-02-20T21:51:31.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 674188288 bytes 2026-02-20T21:51:31.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 707686400 bytes 2026-02-20T21:51:31.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 769894400 bytes 2026-02-20T21:51:31.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 826006528 bytes 2026-02-20T21:51:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 887152640 bytes 2026-02-20T21:51:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 945672192 bytes 2026-02-20T21:51:31.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 965451776 bytes 2026-02-20T21:51:31.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 984667136 bytes 2026-02-20T21:51:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1007851520 bytes 2026-02-20T21:51:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1019832320 bytes 2026-02-20T21:51:31.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1104640000 bytes 2026-02-20T21:51:31.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1118465024 bytes 2026-02-20T21:51:31.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1224090624 bytes 2026-02-20T21:51:31.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1281463296 bytes 2026-02-20T21:51:31.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1334341632 bytes 2026-02-20T21:51:31.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1414760448 bytes 2026-02-20T21:51:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1521176576 bytes 2026-02-20T21:51:32.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1540065280 bytes 2026-02-20T21:51:32.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1595539456 bytes 2026-02-20T21:51:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1660186624 bytes 2026-02-20T21:51:32.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1677069312 bytes 2026-02-20T21:51:32.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1740976128 bytes 2026-02-20T21:51:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1769290752 bytes 2026-02-20T21:51:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1790317568 bytes 2026-02-20T21:51:32.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1869048832 bytes 2026-02-20T21:51:32.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1901915136 bytes 2026-02-20T21:51:32.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1958117376 bytes 2026-02-20T21:51:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 2069107712 bytes 2026-02-20T21:51:32.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 2140010496 bytes 2026-02-20T21:51:32.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 2176461824 bytes 2026-02-20T21:51:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2230025216 bytes 2026-02-20T21:51:32.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2251443200 bytes 2026-02-20T21:51:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2295329792 bytes 2026-02-20T21:51:32.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2330229760 bytes 2026-02-20T21:51:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2374782976 bytes 2026-02-20T21:51:33.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2422508544 bytes 2026-02-20T21:51:33.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2466720768 bytes 2026-02-20T21:51:33.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2528404480 bytes 2026-02-20T21:51:33.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2578598912 bytes 2026-02-20T21:51:33.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2592716800 bytes 2026-02-20T21:51:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2648006656 bytes 2026-02-20T21:51:33.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2716550144 bytes 2026-02-20T21:51:33.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2783874048 bytes 2026-02-20T21:51:33.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2821838848 bytes 2026-02-20T21:51:33.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2867801088 bytes 2026-02-20T21:51:33.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2958184448 bytes 2026-02-20T21:51:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 3028109312 bytes 2026-02-20T21:51:33.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 3092173824 bytes 2026-02-20T21:51:33.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 3111321600 bytes 2026-02-20T21:51:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 3174244352 bytes 2026-02-20T21:51:34.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 3221121024 bytes 2026-02-20T21:51:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 3283187712 bytes 2026-02-20T21:51:34.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3305636864 bytes 2026-02-20T21:51:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3356284928 bytes 2026-02-20T21:51:34.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3399701504 bytes 2026-02-20T21:51:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3407724544 bytes 2026-02-20T21:51:34.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3478223872 bytes 2026-02-20T21:51:34.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3600474112 bytes 2026-02-20T21:51:34.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3653875712 bytes 2026-02-20T21:51:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3699649536 bytes 2026-02-20T21:51:34.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3718684672 bytes 2026-02-20T21:51:34.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3740922880 bytes 2026-02-20T21:51:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3796564992 bytes 2026-02-20T21:51:34.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3810170880 bytes 2026-02-20T21:51:34.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3879484416 bytes 2026-02-20T21:51:34.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3904557056 bytes 2026-02-20T21:51:34.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3949091840 bytes 2026-02-20T21:51:34.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 4042113024 bytes 2026-02-20T21:51:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 4122253312 bytes 2026-02-20T21:51:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 4185693184 bytes 2026-02-20T21:51:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 4204078080 bytes 2026-02-20T21:51:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 4265962496 bytes 2026-02-20T21:51:35.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 4325032960 bytes 2026-02-20T21:51:35.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 4325644288 bytes 2026-02-20T21:51:35.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 4348085248 bytes 2026-02-20T21:51:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 4362120192 bytes 2026-02-20T21:51:35.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4452160512 bytes 2026-02-20T21:51:35.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4465316864 bytes 2026-02-20T21:51:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4476371968 bytes 2026-02-20T21:51:35.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4557470720 bytes 2026-02-20T21:51:35.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4628494336 bytes 2026-02-20T21:51:35.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4694335488 bytes 2026-02-20T21:51:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4724672512 bytes 2026-02-20T21:51:35.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4821405696 bytes 2026-02-20T21:51:35.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4850847744 bytes 2026-02-20T21:51:35.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4925919232 bytes 2026-02-20T21:51:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4974013440 bytes 2026-02-20T21:51:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 5067830272 bytes 2026-02-20T21:51:35.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 5108923392 bytes 2026-02-20T21:51:36.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 5179916288 bytes 2026-02-20T21:51:36.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 5216026624 bytes 2026-02-20T21:51:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 5245602816 bytes 2026-02-20T21:51:36.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 5262778368 bytes 2026-02-20T21:51:36.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 5286803456 bytes 2026-02-20T21:51:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 5356864512 bytes 2026-02-20T21:51:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 5382763520 bytes 2026-02-20T21:51:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 5461007360 bytes 2026-02-20T21:51:36.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 5524570112 bytes 2026-02-20T21:51:36.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5574320128 bytes 2026-02-20T21:51:36.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5574925312 bytes 2026-02-20T21:51:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5638404096 bytes 2026-02-20T21:51:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5639049216 bytes 2026-02-20T21:51:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5688823808 bytes 2026-02-20T21:51:36.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5785230336 bytes 2026-02-20T21:51:36.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5842799616 bytes 2026-02-20T21:51:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5889560576 bytes 2026-02-20T21:51:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5900760064 bytes 2026-02-20T21:51:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5934942208 bytes 2026-02-20T21:51:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5993547776 bytes 2026-02-20T21:51:36.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 6035933184 bytes 2026-02-20T21:51:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 6068530176 bytes 2026-02-20T21:51:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 6111723520 bytes 2026-02-20T21:51:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 6140623872 bytes 2026-02-20T21:51:37.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 6187639808 bytes 2026-02-20T21:51:37.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 6224451584 bytes 2026-02-20T21:51:37.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 6254790656 bytes 2026-02-20T21:51:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 6333854720 bytes 2026-02-20T21:51:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 6337384448 bytes 2026-02-20T21:51:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 6338006016 bytes 2026-02-20T21:51:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 6375207936 bytes 2026-02-20T21:51:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 6383827968 bytes 2026-02-20T21:51:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 6419656704 bytes 2026-02-20T21:51:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 6437011456 bytes 2026-02-20T21:51:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 6475701248 bytes 2026-02-20T21:51:37.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 6531919872 bytes 2026-02-20T21:51:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 6588713984 bytes 2026-02-20T21:51:37.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6607896576 bytes 2026-02-20T21:51:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6636022784 bytes 2026-02-20T21:51:37.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6664691712 bytes 2026-02-20T21:51:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6723740672 bytes 2026-02-20T21:51:37.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6753512448 bytes 2026-02-20T21:51:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6792498176 bytes 2026-02-20T21:51:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6821385216 bytes 2026-02-20T21:51:38.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6850875392 bytes 2026-02-20T21:51:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6871078912 bytes 2026-02-20T21:51:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6893561856 bytes 2026-02-20T21:51:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6937209856 bytes 2026-02-20T21:51:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6970461184 bytes 2026-02-20T21:51:38.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6999411712 bytes 2026-02-20T21:51:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 7089432576 bytes 2026-02-20T21:51:38.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 7137378304 bytes 2026-02-20T21:51:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 7188477952 bytes 2026-02-20T21:51:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 7209229312 bytes 2026-02-20T21:51:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 7240888320 bytes 2026-02-20T21:51:38.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 7307190272 bytes 2026-02-20T21:51:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 7307803648 bytes 2026-02-20T21:51:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 7327043584 bytes 2026-02-20T21:51:38.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 7327699968 bytes 2026-02-20T21:51:38.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 7350281216 bytes 2026-02-20T21:51:38.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7410155520 bytes 2026-02-20T21:51:38.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7503875072 bytes 2026-02-20T21:51:38.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7562748928 bytes 2026-02-20T21:51:38.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7601963008 bytes 2026-02-20T21:51:39.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7669277696 bytes 2026-02-20T21:51:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7688682496 bytes 2026-02-20T21:51:39.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7744694272 bytes 2026-02-20T21:51:39.193 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-20T21:51:39.193 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-20T21:51:39.193 INFO:tasks.rbd_mirror_thrash:waiting for 4.817359675241572 secs before reviving daemons 2026-02-20T21:51:39.194 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T21:51:39.191+0000 7fe8f2069640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 23944) UID: 1000 2026-02-20T21:51:39.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7877894144 bytes 2026-02-20T21:51:39.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7919282176 bytes 2026-02-20T21:51:39.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 8013489152 bytes 2026-02-20T21:51:39.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 8048477184 bytes 2026-02-20T21:51:39.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 8100458496 bytes 2026-02-20T21:51:39.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 8151475200 bytes 2026-02-20T21:51:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 8202896384 bytes 2026-02-20T21:51:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 8266978304 bytes 2026-02-20T21:51:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 8286851072 bytes 2026-02-20T21:51:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 8363350016 bytes 2026-02-20T21:51:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 8424972288 bytes 2026-02-20T21:51:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 8442008576 bytes 2026-02-20T21:51:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 8477388800 bytes 2026-02-20T21:51:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 8583199744 bytes 2026-02-20T21:51:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 8670492672 bytes 2026-02-20T21:51:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 8740737024 bytes 2026-02-20T21:51:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 8753511424 bytes 2026-02-20T21:51:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 8822885376 bytes 2026-02-20T21:51:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 8832206848 bytes 2026-02-20T21:51:40.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8863332352 bytes 2026-02-20T21:51:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8863943680 bytes 2026-02-20T21:51:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8909231104 bytes 2026-02-20T21:51:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 9016482816 bytes 2026-02-20T21:51:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 9066288128 bytes 2026-02-20T21:51:40.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 9080372224 bytes 2026-02-20T21:51:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 9100070912 bytes 2026-02-20T21:51:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 9153470464 bytes 2026-02-20T21:51:40.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 9218811904 bytes 2026-02-20T21:51:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 9280758784 bytes 2026-02-20T21:51:40.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 9283597312 bytes 2026-02-20T21:51:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 9323667456 bytes 2026-02-20T21:51:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 9364308992 bytes 2026-02-20T21:51:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 9439789056 bytes 2026-02-20T21:51:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 9454586880 bytes 2026-02-20T21:51:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 9579504640 bytes 2026-02-20T21:51:41.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 9580111872 bytes 2026-02-20T21:51:41.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 9649671168 bytes 2026-02-20T21:51:41.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 9683884032 bytes 2026-02-20T21:51:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 9704679424 bytes 2026-02-20T21:51:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 9745354752 bytes 2026-02-20T21:51:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 9774213120 bytes 2026-02-20T21:51:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 9792370688 bytes 2026-02-20T21:51:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9796030464 bytes 2026-02-20T21:51:41.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9855647744 bytes 2026-02-20T21:51:41.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9898153984 bytes 2026-02-20T21:51:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9933952000 bytes 2026-02-20T21:51:41.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:51:41.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_23 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 snapshot 128M 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_23 128M 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:51:41.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:51:41.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:51:41.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:41.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_23 2026-02-20T21:51:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:51:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:51:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_23 snapshot 2026-02-20T21:51:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:51:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_23 snapshot 2026-02-20T21:51:42.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:51:42.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_23 2026-02-20T21:51:42.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_23': 2026-02-20T21:51:42.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:51:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:51:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:51:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 138432e68711 2026-02-20T21:51:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.138432e68711 2026-02-20T21:51:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:51:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:51:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:51:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:51:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:41 2026 2026-02-20T21:51:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:41 2026 2026-02-20T21:51:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:41 2026 2026-02-20T21:51:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:51:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:51:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 19082fd1-184d-4597-821e-115e896cdf0f 2026-02-20T21:51:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:51:42.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:51:42.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_23 2026-02-20T21:51:42.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:42.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:42.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:51:42.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:51:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:51:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:51:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:51:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 80374784 bytes 2026-02-20T21:51:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 92605440 bytes 2026-02-20T21:51:42.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 146347008 bytes 2026-02-20T21:51:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 170212352 bytes 2026-02-20T21:51:42.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 216609792 bytes 2026-02-20T21:51:42.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 259648512 bytes 2026-02-20T21:51:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 290804736 bytes 2026-02-20T21:51:42.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 332309504 bytes 2026-02-20T21:51:43.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 360592384 bytes 2026-02-20T21:51:43.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 364177408 bytes 2026-02-20T21:51:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 446271488 bytes 2026-02-20T21:51:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 491082752 bytes 2026-02-20T21:51:43.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 518188032 bytes 2026-02-20T21:51:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 522121216 bytes 2026-02-20T21:51:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 543918080 bytes 2026-02-20T21:51:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 593438720 bytes 2026-02-20T21:51:43.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 603491328 bytes 2026-02-20T21:51:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 668339200 bytes 2026-02-20T21:51:43.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 689120256 bytes 2026-02-20T21:51:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 758243328 bytes 2026-02-20T21:51:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 803617792 bytes 2026-02-20T21:51:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 887152640 bytes 2026-02-20T21:51:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 921236480 bytes 2026-02-20T21:51:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 982531072 bytes 2026-02-20T21:51:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1049854976 bytes 2026-02-20T21:51:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1067468800 bytes 2026-02-20T21:51:43.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1129509888 bytes 2026-02-20T21:51:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1132778496 bytes 2026-02-20T21:51:43.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1155279872 bytes 2026-02-20T21:51:44.011 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-20T21:51:44.012 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:51:44.012 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-20T21:51:44.013 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T21:51:44.013 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T21:51:44.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1212839936 bytes 2026-02-20T21:51:44.016 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T21:51:44.016 INFO:tasks.rbd_mirror_thrash:waiting for 103.50692919693587 secs before thrashing 2026-02-20T21:51:44.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1259656192 bytes 2026-02-20T21:51:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1285733376 bytes 2026-02-20T21:51:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1346455552 bytes 2026-02-20T21:51:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1377636352 bytes 2026-02-20T21:51:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1388637184 bytes 2026-02-20T21:51:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1475358720 bytes 2026-02-20T21:51:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1537973248 bytes 2026-02-20T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1579235328 bytes 2026-02-20T21:51:44.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1642251264 bytes 2026-02-20T21:51:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1643386880 bytes 2026-02-20T21:51:44.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1698447360 bytes 2026-02-20T21:51:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1811904512 bytes 2026-02-20T21:51:44.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1862986752 bytes 2026-02-20T21:51:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1911146496 bytes 2026-02-20T21:51:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1940980736 bytes 2026-02-20T21:51:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1941584896 bytes 2026-02-20T21:51:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2017565696 bytes 2026-02-20T21:51:44.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2060331008 bytes 2026-02-20T21:51:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2112707584 bytes 2026-02-20T21:51:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2168050688 bytes 2026-02-20T21:51:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2187812864 bytes 2026-02-20T21:51:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2246036480 bytes 2026-02-20T21:51:45.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2290851840 bytes 2026-02-20T21:51:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2368733184 bytes 2026-02-20T21:51:45.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2404819968 bytes 2026-02-20T21:51:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2417193984 bytes 2026-02-20T21:51:45.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2417825792 bytes 2026-02-20T21:51:45.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2442068992 bytes 2026-02-20T21:51:45.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2509166592 bytes 2026-02-20T21:51:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2516965376 bytes 2026-02-20T21:51:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2587404288 bytes 2026-02-20T21:51:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2593969152 bytes 2026-02-20T21:51:45.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2600048640 bytes 2026-02-20T21:51:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2621041664 bytes 2026-02-20T21:51:45.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2669991936 bytes 2026-02-20T21:51:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2765082624 bytes 2026-02-20T21:51:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2778428416 bytes 2026-02-20T21:51:45.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2800779264 bytes 2026-02-20T21:51:45.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2824507392 bytes 2026-02-20T21:51:45.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2843913216 bytes 2026-02-20T21:51:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2868904960 bytes 2026-02-20T21:51:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2891184128 bytes 2026-02-20T21:51:46.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2920690688 bytes 2026-02-20T21:51:46.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2959730688 bytes 2026-02-20T21:51:46.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2960983040 bytes 2026-02-20T21:51:46.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3038819328 bytes 2026-02-20T21:51:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3055399936 bytes 2026-02-20T21:51:46.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3080305664 bytes 2026-02-20T21:51:46.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3159518208 bytes 2026-02-20T21:51:46.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3205297152 bytes 2026-02-20T21:51:46.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3237632000 bytes 2026-02-20T21:51:46.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3273396224 bytes 2026-02-20T21:51:46.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3343997952 bytes 2026-02-20T21:51:46.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3369622528 bytes 2026-02-20T21:51:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3400413184 bytes 2026-02-20T21:51:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3401038848 bytes 2026-02-20T21:51:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3456393216 bytes 2026-02-20T21:51:46.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3490771968 bytes 2026-02-20T21:51:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3552303104 bytes 2026-02-20T21:51:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3654392832 bytes 2026-02-20T21:51:46.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3708327936 bytes 2026-02-20T21:51:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3723141120 bytes 2026-02-20T21:51:47.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3775333376 bytes 2026-02-20T21:51:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3841340416 bytes 2026-02-20T21:51:47.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3892126720 bytes 2026-02-20T21:51:47.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4002486272 bytes 2026-02-20T21:51:47.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4003093504 bytes 2026-02-20T21:51:47.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4023245824 bytes 2026-02-20T21:51:47.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4039936000 bytes 2026-02-20T21:51:47.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4085795840 bytes 2026-02-20T21:51:47.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4113304576 bytes 2026-02-20T21:51:47.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4213089280 bytes 2026-02-20T21:51:47.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4282125312 bytes 2026-02-20T21:51:47.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4282748928 bytes 2026-02-20T21:51:47.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4371210240 bytes 2026-02-20T21:51:47.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4387540992 bytes 2026-02-20T21:51:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4388175872 bytes 2026-02-20T21:51:47.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4429569024 bytes 2026-02-20T21:51:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4477365248 bytes 2026-02-20T21:51:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4513002496 bytes 2026-02-20T21:51:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4589946880 bytes 2026-02-20T21:51:48.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4646666240 bytes 2026-02-20T21:51:48.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4718629888 bytes 2026-02-20T21:51:48.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4776121344 bytes 2026-02-20T21:51:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4798403584 bytes 2026-02-20T21:51:48.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4828394496 bytes 2026-02-20T21:51:48.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4877859840 bytes 2026-02-20T21:51:48.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4950274048 bytes 2026-02-20T21:51:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5014325248 bytes 2026-02-20T21:51:48.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5079662592 bytes 2026-02-20T21:51:48.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:51:48.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:51:48.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_24 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 snapshot 128M 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:51:48.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_24 128M 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:51:48.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:51:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:51:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:48.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_24 2026-02-20T21:51:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:51:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:51:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_24 snapshot 2026-02-20T21:51:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:51:48.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:48.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_24 snapshot 2026-02-20T21:51:49.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:51:49.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_24 2026-02-20T21:51:49.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_24': 2026-02-20T21:51:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:51:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:51:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:51:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13958197141c 2026-02-20T21:51:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13958197141c 2026-02-20T21:51:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:51:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:51:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:51:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:51:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:48 2026 2026-02-20T21:51:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:48 2026 2026-02-20T21:51:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:48 2026 2026-02-20T21:51:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:51:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:51:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: af49b5c9-f8fe-41c3-a35c-c2ad69c2bd7f 2026-02-20T21:51:49.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:51:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:51:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_24 2026-02-20T21:51:49.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:49.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:49.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:51:49.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:51:49.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T21:51:49.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:51:49.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:51:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 28229632 bytes 2026-02-20T21:51:49.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 102215680 bytes 2026-02-20T21:51:49.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 129182720 bytes 2026-02-20T21:51:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 129769472 bytes 2026-02-20T21:51:49.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 142302208 bytes 2026-02-20T21:51:49.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 230760448 bytes 2026-02-20T21:51:49.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 272659456 bytes 2026-02-20T21:51:49.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 323156992 bytes 2026-02-20T21:51:50.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 347304960 bytes 2026-02-20T21:51:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 391173120 bytes 2026-02-20T21:51:50.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 413649920 bytes 2026-02-20T21:51:50.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 434971648 bytes 2026-02-20T21:51:50.207 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 509058048 bytes 2026-02-20T21:51:50.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 535223296 bytes 2026-02-20T21:51:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 536637440 bytes 2026-02-20T21:51:50.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 555966464 bytes 2026-02-20T21:51:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 596869120 bytes 2026-02-20T21:51:50.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 626838528 bytes 2026-02-20T21:51:50.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 664269824 bytes 2026-02-20T21:51:50.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 722085888 bytes 2026-02-20T21:51:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 735191040 bytes 2026-02-20T21:51:50.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 735788032 bytes 2026-02-20T21:51:50.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 759281664 bytes 2026-02-20T21:51:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 779936768 bytes 2026-02-20T21:51:50.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 780570624 bytes 2026-02-20T21:51:50.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 888014848 bytes 2026-02-20T21:51:50.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 920549376 bytes 2026-02-20T21:51:50.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 942196736 bytes 2026-02-20T21:51:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 989250560 bytes 2026-02-20T21:51:50.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1050112000 bytes 2026-02-20T21:51:51.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1097753600 bytes 2026-02-20T21:51:51.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1174260736 bytes 2026-02-20T21:51:51.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1215560704 bytes 2026-02-20T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1234742272 bytes 2026-02-20T21:51:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1236595712 bytes 2026-02-20T21:51:51.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1316714496 bytes 2026-02-20T21:51:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1359425536 bytes 2026-02-20T21:51:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1404189696 bytes 2026-02-20T21:51:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1414401024 bytes 2026-02-20T21:51:51.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1502502912 bytes 2026-02-20T21:51:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1544916992 bytes 2026-02-20T21:51:51.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1588853760 bytes 2026-02-20T21:51:51.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1621389312 bytes 2026-02-20T21:51:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1668946944 bytes 2026-02-20T21:51:51.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1694376960 bytes 2026-02-20T21:51:51.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1714934784 bytes 2026-02-20T21:51:51.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1753725952 bytes 2026-02-20T21:51:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1790671872 bytes 2026-02-20T21:51:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1845143552 bytes 2026-02-20T21:51:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1846141952 bytes 2026-02-20T21:51:51.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 1874315264 bytes 2026-02-20T21:51:51.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 1901008896 bytes 2026-02-20T21:51:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 1928767488 bytes 2026-02-20T21:51:52.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 1996226560 bytes 2026-02-20T21:51:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2072144896 bytes 2026-02-20T21:51:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2131814400 bytes 2026-02-20T21:51:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2172267520 bytes 2026-02-20T21:51:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2267716608 bytes 2026-02-20T21:51:52.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2337305600 bytes 2026-02-20T21:51:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2456120320 bytes 2026-02-20T21:51:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2469286912 bytes 2026-02-20T21:51:52.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2503161856 bytes 2026-02-20T21:51:52.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2526376960 bytes 2026-02-20T21:51:52.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2554579968 bytes 2026-02-20T21:51:52.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2573370368 bytes 2026-02-20T21:51:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2603311104 bytes 2026-02-20T21:51:52.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2638697472 bytes 2026-02-20T21:51:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2701064192 bytes 2026-02-20T21:51:52.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2713988096 bytes 2026-02-20T21:51:52.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2759383040 bytes 2026-02-20T21:51:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2776108032 bytes 2026-02-20T21:51:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2784391168 bytes 2026-02-20T21:51:53.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2855016448 bytes 2026-02-20T21:51:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2893592576 bytes 2026-02-20T21:51:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2959902720 bytes 2026-02-20T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3001953280 bytes 2026-02-20T21:51:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3034945536 bytes 2026-02-20T21:51:53.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3046267904 bytes 2026-02-20T21:51:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3081639936 bytes 2026-02-20T21:51:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3142472704 bytes 2026-02-20T21:51:53.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3185569792 bytes 2026-02-20T21:51:53.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3265589248 bytes 2026-02-20T21:51:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3292000256 bytes 2026-02-20T21:51:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3318301696 bytes 2026-02-20T21:51:53.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3321402368 bytes 2026-02-20T21:51:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3348104192 bytes 2026-02-20T21:51:53.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3357391872 bytes 2026-02-20T21:51:53.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3391970304 bytes 2026-02-20T21:51:53.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3413895168 bytes 2026-02-20T21:51:53.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3466142720 bytes 2026-02-20T21:51:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3535286272 bytes 2026-02-20T21:51:53.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3573086208 bytes 2026-02-20T21:51:53.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3594628096 bytes 2026-02-20T21:51:54.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3690724352 bytes 2026-02-20T21:51:54.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3691346944 bytes 2026-02-20T21:51:54.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3745411072 bytes 2026-02-20T21:51:54.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3747905536 bytes 2026-02-20T21:51:54.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3809377280 bytes 2026-02-20T21:51:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3858464768 bytes 2026-02-20T21:51:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3944629248 bytes 2026-02-20T21:51:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4016633856 bytes 2026-02-20T21:51:54.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4031212544 bytes 2026-02-20T21:51:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4056988672 bytes 2026-02-20T21:51:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4132795392 bytes 2026-02-20T21:51:54.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4178143232 bytes 2026-02-20T21:51:54.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4246489088 bytes 2026-02-20T21:51:54.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4265139200 bytes 2026-02-20T21:51:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4265765888 bytes 2026-02-20T21:51:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4283299840 bytes 2026-02-20T21:51:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4371570688 bytes 2026-02-20T21:51:54.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4417120256 bytes 2026-02-20T21:51:54.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4528345088 bytes 2026-02-20T21:51:54.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4577738752 bytes 2026-02-20T21:51:55.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4606030848 bytes 2026-02-20T21:51:55.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4615667712 bytes 2026-02-20T21:51:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4693602304 bytes 2026-02-20T21:51:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4759096320 bytes 2026-02-20T21:51:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4800995328 bytes 2026-02-20T21:51:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4865218560 bytes 2026-02-20T21:51:55.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4867554304 bytes 2026-02-20T21:51:55.384 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4906398720 bytes 2026-02-20T21:51:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4958075904 bytes 2026-02-20T21:51:55.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4969794560 bytes 2026-02-20T21:51:55.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5047452672 bytes 2026-02-20T21:51:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5089197056 bytes 2026-02-20T21:51:55.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5124972544 bytes 2026-02-20T21:51:55.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5197048832 bytes 2026-02-20T21:51:55.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5213244416 bytes 2026-02-20T21:51:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5279644672 bytes 2026-02-20T21:51:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5312419840 bytes 2026-02-20T21:51:55.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5342577664 bytes 2026-02-20T21:51:55.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5383959552 bytes 2026-02-20T21:51:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5386721280 bytes 2026-02-20T21:51:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5395198976 bytes 2026-02-20T21:51:56.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5414445056 bytes 2026-02-20T21:51:56.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5456311296 bytes 2026-02-20T21:51:56.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5495313408 bytes 2026-02-20T21:51:56.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5548207104 bytes 2026-02-20T21:51:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5566115840 bytes 2026-02-20T21:51:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5593006080 bytes 2026-02-20T21:51:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5662080000 bytes 2026-02-20T21:51:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5706812416 bytes 2026-02-20T21:51:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5781310464 bytes 2026-02-20T21:51:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5803739136 bytes 2026-02-20T21:51:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5821301760 bytes 2026-02-20T21:51:56.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5863842816 bytes 2026-02-20T21:51:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5888934912 bytes 2026-02-20T21:51:56.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5916984320 bytes 2026-02-20T21:51:56.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5975197696 bytes 2026-02-20T21:51:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5983475712 bytes 2026-02-20T21:51:56.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6030573568 bytes 2026-02-20T21:51:56.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6102389760 bytes 2026-02-20T21:51:56.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6146937856 bytes 2026-02-20T21:51:56.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6160446464 bytes 2026-02-20T21:51:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6210430976 bytes 2026-02-20T21:51:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6288505856 bytes 2026-02-20T21:51:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6338299904 bytes 2026-02-20T21:51:57.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6390501376 bytes 2026-02-20T21:51:57.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6402297856 bytes 2026-02-20T21:51:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6468510720 bytes 2026-02-20T21:51:57.340 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6547152896 bytes 2026-02-20T21:51:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6547977216 bytes 2026-02-20T21:51:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6587878400 bytes 2026-02-20T21:51:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6663890944 bytes 2026-02-20T21:51:57.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6722680832 bytes 2026-02-20T21:51:57.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6781135872 bytes 2026-02-20T21:51:57.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6822976512 bytes 2026-02-20T21:51:57.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6871412736 bytes 2026-02-20T21:51:57.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6937775104 bytes 2026-02-20T21:51:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6953712640 bytes 2026-02-20T21:51:57.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6990539776 bytes 2026-02-20T21:51:57.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7064183808 bytes 2026-02-20T21:51:57.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7081596928 bytes 2026-02-20T21:51:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7137050624 bytes 2026-02-20T21:51:58.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7181828096 bytes 2026-02-20T21:51:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7245046784 bytes 2026-02-20T21:51:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7282930688 bytes 2026-02-20T21:51:58.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7329637376 bytes 2026-02-20T21:51:58.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7411357696 bytes 2026-02-20T21:51:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7466330112 bytes 2026-02-20T21:51:58.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7479603200 bytes 2026-02-20T21:51:58.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7540643840 bytes 2026-02-20T21:51:58.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7559892992 bytes 2026-02-20T21:51:58.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7591980032 bytes 2026-02-20T21:51:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:51:58.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:51:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:51:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:51:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_25 2026-02-20T21:51:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 snapshot 128M 2026-02-20T21:51:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:58.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T21:51:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:51:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_25 128M 2026-02-20T21:51:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:58.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:58.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:58.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:58.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T21:51:58.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:58.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:51:58.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:51:58.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:51:58.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:51:58.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_25 2026-02-20T21:51:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:51:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:51:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:51:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_25 snapshot 2026-02-20T21:51:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T21:51:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:51:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_25 snapshot 2026-02-20T21:51:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:51:59.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_25 2026-02-20T21:51:59.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_25': 2026-02-20T21:51:59.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:51:59.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:51:59.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:51:59.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13a12172d034 2026-02-20T21:51:59.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13a12172d034 2026-02-20T21:51:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:51:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:51:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:51:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:51:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:58 2026 2026-02-20T21:51:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:58 2026 2026-02-20T21:51:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:58 2026 2026-02-20T21:51:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:51:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:51:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 013e07d3-729c-41fe-bfbb-6be668748dac 2026-02-20T21:51:59.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:51:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:51:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_25 2026-02-20T21:51:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:51:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:51:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T21:51:59.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=14 2026-02-20T21:51:59.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:51:59.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:51:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 24616960 bytes 2026-02-20T21:51:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 94764032 bytes 2026-02-20T21:51:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 140736512 bytes 2026-02-20T21:51:59.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 152225792 bytes 2026-02-20T21:51:59.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 182730752 bytes 2026-02-20T21:51:59.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 260195328 bytes 2026-02-20T21:51:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 302593024 bytes 2026-02-20T21:51:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 314503168 bytes 2026-02-20T21:52:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 347864064 bytes 2026-02-20T21:52:00.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 397310976 bytes 2026-02-20T21:52:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 407040000 bytes 2026-02-20T21:52:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 447046656 bytes 2026-02-20T21:52:00.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 471769088 bytes 2026-02-20T21:52:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 498201600 bytes 2026-02-20T21:52:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 531329024 bytes 2026-02-20T21:52:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 572624896 bytes 2026-02-20T21:52:00.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 601183232 bytes 2026-02-20T21:52:00.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 660215808 bytes 2026-02-20T21:52:00.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 673509376 bytes 2026-02-20T21:52:00.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 699281408 bytes 2026-02-20T21:52:00.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 783092736 bytes 2026-02-20T21:52:00.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 812314624 bytes 2026-02-20T21:52:00.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 861912064 bytes 2026-02-20T21:52:00.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 886125568 bytes 2026-02-20T21:52:00.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 910740480 bytes 2026-02-20T21:52:00.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 957696000 bytes 2026-02-20T21:52:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 990204928 bytes 2026-02-20T21:52:00.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1029377024 bytes 2026-02-20T21:52:00.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1044189184 bytes 2026-02-20T21:52:00.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1103221760 bytes 2026-02-20T21:52:01.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1134642176 bytes 2026-02-20T21:52:01.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1183902720 bytes 2026-02-20T21:52:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1206110208 bytes 2026-02-20T21:52:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1253453824 bytes 2026-02-20T21:52:01.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1274676224 bytes 2026-02-20T21:52:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1326162944 bytes 2026-02-20T21:52:01.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1358036992 bytes 2026-02-20T21:52:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1403259904 bytes 2026-02-20T21:52:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1452070912 bytes 2026-02-20T21:52:01.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1492141056 bytes 2026-02-20T21:52:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1553276928 bytes 2026-02-20T21:52:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1601636352 bytes 2026-02-20T21:52:01.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1665060864 bytes 2026-02-20T21:52:01.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1687224320 bytes 2026-02-20T21:52:01.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1687806976 bytes 2026-02-20T21:52:01.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1727360000 bytes 2026-02-20T21:52:01.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1800558592 bytes 2026-02-20T21:52:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1851383808 bytes 2026-02-20T21:52:01.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1919122432 bytes 2026-02-20T21:52:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1920920576 bytes 2026-02-20T21:52:01.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 1952214016 bytes 2026-02-20T21:52:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 1963325440 bytes 2026-02-20T21:52:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2029490176 bytes 2026-02-20T21:52:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2043829248 bytes 2026-02-20T21:52:02.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2044440576 bytes 2026-02-20T21:52:02.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2085620736 bytes 2026-02-20T21:52:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2215815168 bytes 2026-02-20T21:52:02.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2230158336 bytes 2026-02-20T21:52:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2280562688 bytes 2026-02-20T21:52:02.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2293858304 bytes 2026-02-20T21:52:02.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2343597056 bytes 2026-02-20T21:52:02.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2372477952 bytes 2026-02-20T21:52:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2468216832 bytes 2026-02-20T21:52:02.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2477493248 bytes 2026-02-20T21:52:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2534692864 bytes 2026-02-20T21:52:02.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2535328768 bytes 2026-02-20T21:52:02.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2553524224 bytes 2026-02-20T21:52:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2575738880 bytes 2026-02-20T21:52:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2608190464 bytes 2026-02-20T21:52:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2728358912 bytes 2026-02-20T21:52:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2793519104 bytes 2026-02-20T21:52:02.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2794125312 bytes 2026-02-20T21:52:03.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2800986112 bytes 2026-02-20T21:52:03.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2869956608 bytes 2026-02-20T21:52:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2876022784 bytes 2026-02-20T21:52:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 2923103232 bytes 2026-02-20T21:52:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 2923718656 bytes 2026-02-20T21:52:03.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 2996315136 bytes 2026-02-20T21:52:03.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3051038720 bytes 2026-02-20T21:52:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3069644800 bytes 2026-02-20T21:52:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3112286208 bytes 2026-02-20T21:52:03.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3125987328 bytes 2026-02-20T21:52:03.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3182830592 bytes 2026-02-20T21:52:03.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3183459328 bytes 2026-02-20T21:52:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3234177024 bytes 2026-02-20T21:52:03.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3309497344 bytes 2026-02-20T21:52:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3351834624 bytes 2026-02-20T21:52:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3401881600 bytes 2026-02-20T21:52:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3447055360 bytes 2026-02-20T21:52:03.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3495811072 bytes 2026-02-20T21:52:03.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3533892608 bytes 2026-02-20T21:52:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3636632576 bytes 2026-02-20T21:52:03.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3668092928 bytes 2026-02-20T21:52:04.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3708035072 bytes 2026-02-20T21:52:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3726242816 bytes 2026-02-20T21:52:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3755452416 bytes 2026-02-20T21:52:04.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3787564032 bytes 2026-02-20T21:52:04.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3833372672 bytes 2026-02-20T21:52:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3877599232 bytes 2026-02-20T21:52:04.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3895359488 bytes 2026-02-20T21:52:04.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 3922720768 bytes 2026-02-20T21:52:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 3979667456 bytes 2026-02-20T21:52:04.427 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:52:04.423+0000 7fa5e2658640 -1 reset not still connected to 0x55b91ce07ee0 2026-02-20T21:52:04.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4078168064 bytes 2026-02-20T21:52:04.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4103799808 bytes 2026-02-20T21:52:04.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4157927424 bytes 2026-02-20T21:52:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4189624320 bytes 2026-02-20T21:52:04.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4209031168 bytes 2026-02-20T21:52:04.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4249861120 bytes 2026-02-20T21:52:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4303528960 bytes 2026-02-20T21:52:04.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4354823168 bytes 2026-02-20T21:52:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4386279424 bytes 2026-02-20T21:52:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4407180288 bytes 2026-02-20T21:52:04.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4416196608 bytes 2026-02-20T21:52:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4473606144 bytes 2026-02-20T21:52:05.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4505730048 bytes 2026-02-20T21:52:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4554775552 bytes 2026-02-20T21:52:05.100 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4565239808 bytes 2026-02-20T21:52:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4578995200 bytes 2026-02-20T21:52:05.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4618974208 bytes 2026-02-20T21:52:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4712850432 bytes 2026-02-20T21:52:05.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4751725568 bytes 2026-02-20T21:52:05.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4816480256 bytes 2026-02-20T21:52:05.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4862926848 bytes 2026-02-20T21:52:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4879519744 bytes 2026-02-20T21:52:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4945560576 bytes 2026-02-20T21:52:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 4946192384 bytes 2026-02-20T21:52:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 4982799360 bytes 2026-02-20T21:52:05.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5030417408 bytes 2026-02-20T21:52:05.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5101052928 bytes 2026-02-20T21:52:05.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5154055168 bytes 2026-02-20T21:52:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5173966848 bytes 2026-02-20T21:52:05.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5244987392 bytes 2026-02-20T21:52:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5358013440 bytes 2026-02-20T21:52:05.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5434818560 bytes 2026-02-20T21:52:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5496570880 bytes 2026-02-20T21:52:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5583877120 bytes 2026-02-20T21:52:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5649659904 bytes 2026-02-20T21:52:06.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5650260992 bytes 2026-02-20T21:52:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5711048704 bytes 2026-02-20T21:52:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5752247296 bytes 2026-02-20T21:52:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5836577792 bytes 2026-02-20T21:52:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5851423744 bytes 2026-02-20T21:52:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5898387456 bytes 2026-02-20T21:52:06.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5936908288 bytes 2026-02-20T21:52:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5986204672 bytes 2026-02-20T21:52:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6077974528 bytes 2026-02-20T21:52:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6121766912 bytes 2026-02-20T21:52:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6143040512 bytes 2026-02-20T21:52:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6163603456 bytes 2026-02-20T21:52:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6192362496 bytes 2026-02-20T21:52:06.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6195560448 bytes 2026-02-20T21:52:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6232205312 bytes 2026-02-20T21:52:06.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6280068096 bytes 2026-02-20T21:52:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6316992512 bytes 2026-02-20T21:52:06.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6375166976 bytes 2026-02-20T21:52:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6428410880 bytes 2026-02-20T21:52:07.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6494654464 bytes 2026-02-20T21:52:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6506086400 bytes 2026-02-20T21:52:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6528679936 bytes 2026-02-20T21:52:07.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6608907264 bytes 2026-02-20T21:52:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6620655616 bytes 2026-02-20T21:52:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6674105344 bytes 2026-02-20T21:52:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6695939072 bytes 2026-02-20T21:52:07.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6710450176 bytes 2026-02-20T21:52:07.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6751403008 bytes 2026-02-20T21:52:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6834969600 bytes 2026-02-20T21:52:07.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6906454016 bytes 2026-02-20T21:52:07.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6941216768 bytes 2026-02-20T21:52:07.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6970138624 bytes 2026-02-20T21:52:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7004676096 bytes 2026-02-20T21:52:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7059825664 bytes 2026-02-20T21:52:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7132568576 bytes 2026-02-20T21:52:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7179317248 bytes 2026-02-20T21:52:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7290504192 bytes 2026-02-20T21:52:07.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7307046912 bytes 2026-02-20T21:52:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7330856960 bytes 2026-02-20T21:52:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7361507328 bytes 2026-02-20T21:52:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7373546496 bytes 2026-02-20T21:52:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7407725568 bytes 2026-02-20T21:52:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7458182144 bytes 2026-02-20T21:52:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7472808960 bytes 2026-02-20T21:52:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7531128832 bytes 2026-02-20T21:52:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7550625792 bytes 2026-02-20T21:52:08.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7576528896 bytes 2026-02-20T21:52:08.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7613751296 bytes 2026-02-20T21:52:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7664425984 bytes 2026-02-20T21:52:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7738487808 bytes 2026-02-20T21:52:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7894560768 bytes 2026-02-20T21:52:08.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7916461056 bytes 2026-02-20T21:52:08.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7944077312 bytes 2026-02-20T21:52:08.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7980608512 bytes 2026-02-20T21:52:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8029353984 bytes 2026-02-20T21:52:08.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8078517248 bytes 2026-02-20T21:52:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8115938304 bytes 2026-02-20T21:52:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8136243200 bytes 2026-02-20T21:52:08.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8153142272 bytes 2026-02-20T21:52:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8196703232 bytes 2026-02-20T21:52:09.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8243784704 bytes 2026-02-20T21:52:09.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8306896896 bytes 2026-02-20T21:52:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8341542912 bytes 2026-02-20T21:52:09.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8342151168 bytes 2026-02-20T21:52:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8350615552 bytes 2026-02-20T21:52:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8382670848 bytes 2026-02-20T21:52:09.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8398988288 bytes 2026-02-20T21:52:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8450289664 bytes 2026-02-20T21:52:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8472899584 bytes 2026-02-20T21:52:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8498497536 bytes 2026-02-20T21:52:09.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8572298240 bytes 2026-02-20T21:52:09.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8590703616 bytes 2026-02-20T21:52:09.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8604476416 bytes 2026-02-20T21:52:09.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8640720896 bytes 2026-02-20T21:52:09.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8660843520 bytes 2026-02-20T21:52:09.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8692043776 bytes 2026-02-20T21:52:09.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8773561344 bytes 2026-02-20T21:52:09.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8799143936 bytes 2026-02-20T21:52:09.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 8825665536 bytes 2026-02-20T21:52:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 8894650368 bytes 2026-02-20T21:52:09.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 8960274432 bytes 2026-02-20T21:52:10.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9049743360 bytes 2026-02-20T21:52:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9069637632 bytes 2026-02-20T21:52:10.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9088399360 bytes 2026-02-20T21:52:10.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9132602368 bytes 2026-02-20T21:52:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9161718784 bytes 2026-02-20T21:52:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9189301248 bytes 2026-02-20T21:52:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9245878272 bytes 2026-02-20T21:52:10.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9302163456 bytes 2026-02-20T21:52:10.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9335481344 bytes 2026-02-20T21:52:10.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9358055424 bytes 2026-02-20T21:52:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9412315136 bytes 2026-02-20T21:52:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9477991424 bytes 2026-02-20T21:52:10.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9528670208 bytes 2026-02-20T21:52:10.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9589881856 bytes 2026-02-20T21:52:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9680729088 bytes 2026-02-20T21:52:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9703766016 bytes 2026-02-20T21:52:10.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9760308224 bytes 2026-02-20T21:52:10.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 9855065088 bytes 2026-02-20T21:52:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 9892889600 bytes 2026-02-20T21:52:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 9949810688 bytes 2026-02-20T21:52:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 10031097856 bytes 2026-02-20T21:52:11.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 10060422144 bytes 2026-02-20T21:52:11.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 10136085504 bytes 2026-02-20T21:52:11.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10175816704 bytes 2026-02-20T21:52:11.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 10219650048 bytes 2026-02-20T21:52:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 10220273664 bytes 2026-02-20T21:52:11.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10243944448 bytes 2026-02-20T21:52:11.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10270264320 bytes 2026-02-20T21:52:11.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10343800832 bytes 2026-02-20T21:52:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10367840256 bytes 2026-02-20T21:52:11.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 10436846592 bytes 2026-02-20T21:52:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 10519923712 bytes 2026-02-20T21:52:11.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25100 IOs, 10537140224 bytes 2026-02-20T21:52:11.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25200 IOs, 10588365824 bytes 2026-02-20T21:52:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25300 IOs, 10628122624 bytes 2026-02-20T21:52:11.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25400 IOs, 10673031168 bytes 2026-02-20T21:52:11.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25500 IOs, 10774539264 bytes 2026-02-20T21:52:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25600 IOs, 10810290176 bytes 2026-02-20T21:52:11.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25700 IOs, 10858899456 bytes 2026-02-20T21:52:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25800 IOs, 10872217600 bytes 2026-02-20T21:52:12.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25900 IOs, 10958480384 bytes 2026-02-20T21:52:12.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26000 IOs, 10974100480 bytes 2026-02-20T21:52:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26100 IOs, 11008269312 bytes 2026-02-20T21:52:12.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26200 IOs, 11037880320 bytes 2026-02-20T21:52:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26300 IOs, 11045980160 bytes 2026-02-20T21:52:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26400 IOs, 11085287424 bytes 2026-02-20T21:52:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26500 IOs, 11105401856 bytes 2026-02-20T21:52:12.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26600 IOs, 11157380096 bytes 2026-02-20T21:52:12.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26700 IOs, 11216745472 bytes 2026-02-20T21:52:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26800 IOs, 11269933056 bytes 2026-02-20T21:52:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26900 IOs, 11340496896 bytes 2026-02-20T21:52:12.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27000 IOs, 11435036672 bytes 2026-02-20T21:52:12.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27100 IOs, 11485067264 bytes 2026-02-20T21:52:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27200 IOs, 11520592896 bytes 2026-02-20T21:52:12.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27300 IOs, 11528114176 bytes 2026-02-20T21:52:12.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27400 IOs, 11528687616 bytes 2026-02-20T21:52:12.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27500 IOs, 11551968256 bytes 2026-02-20T21:52:12.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27600 IOs, 11625641984 bytes 2026-02-20T21:52:12.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27700 IOs, 11628830720 bytes 2026-02-20T21:52:12.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27800 IOs, 11672523776 bytes 2026-02-20T21:52:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27900 IOs, 11705213952 bytes 2026-02-20T21:52:13.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28000 IOs, 11790613504 bytes 2026-02-20T21:52:13.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28100 IOs, 11840932864 bytes 2026-02-20T21:52:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28200 IOs, 11868845056 bytes 2026-02-20T21:52:13.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28300 IOs, 11892560896 bytes 2026-02-20T21:52:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28400 IOs, 11919406080 bytes 2026-02-20T21:52:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28500 IOs, 11954206720 bytes 2026-02-20T21:52:13.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28600 IOs, 12078143488 bytes 2026-02-20T21:52:13.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28700 IOs, 12109789184 bytes 2026-02-20T21:52:13.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28800 IOs, 12168402944 bytes 2026-02-20T21:52:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:28900 IOs, 12200013824 bytes 2026-02-20T21:52:13.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:52:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:52:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:52:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:52:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_26 2026-02-20T21:52:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 snapshot 128M 2026-02-20T21:52:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:13.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_26 128M 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:13.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T21:52:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:52:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:52:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:52:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:13.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_26 2026-02-20T21:52:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:52:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:52:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_26 snapshot 2026-02-20T21:52:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T21:52:13.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:13.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_26 snapshot 2026-02-20T21:52:14.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:52:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_26 2026-02-20T21:52:14.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_26': 2026-02-20T21:52:14.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13ad94b8152b 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13ad94b8152b 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:13 2026 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:13 2026 2026-02-20T21:52:14.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:13 2026 2026-02-20T21:52:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:52:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:52:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: eebea35a-dfd0-4c97-aade-b52b3cada4e8 2026-02-20T21:52:14.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:52:14.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:52:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_26 2026-02-20T21:52:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T21:52:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:52:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=5 2026-02-20T21:52:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:52:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:52:14.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 15784960 bytes 2026-02-20T21:52:14.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 21044224 bytes 2026-02-20T21:52:14.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 69935104 bytes 2026-02-20T21:52:14.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 70565888 bytes 2026-02-20T21:52:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 131176448 bytes 2026-02-20T21:52:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 182335488 bytes 2026-02-20T21:52:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 251911168 bytes 2026-02-20T21:52:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 309830656 bytes 2026-02-20T21:52:15.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 327847936 bytes 2026-02-20T21:52:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 389681152 bytes 2026-02-20T21:52:15.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 402150400 bytes 2026-02-20T21:52:15.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 517649408 bytes 2026-02-20T21:52:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 555677696 bytes 2026-02-20T21:52:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 558580736 bytes 2026-02-20T21:52:15.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 618851328 bytes 2026-02-20T21:52:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 718612480 bytes 2026-02-20T21:52:15.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 722820096 bytes 2026-02-20T21:52:15.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 778040320 bytes 2026-02-20T21:52:15.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 788496384 bytes 2026-02-20T21:52:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 891474944 bytes 2026-02-20T21:52:15.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 924670976 bytes 2026-02-20T21:52:15.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 963861504 bytes 2026-02-20T21:52:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 990352384 bytes 2026-02-20T21:52:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1078429696 bytes 2026-02-20T21:52:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1107176448 bytes 2026-02-20T21:52:15.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1128163328 bytes 2026-02-20T21:52:15.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1165582336 bytes 2026-02-20T21:52:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1246131200 bytes 2026-02-20T21:52:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1316069376 bytes 2026-02-20T21:52:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1352846336 bytes 2026-02-20T21:52:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1367038976 bytes 2026-02-20T21:52:16.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1447894016 bytes 2026-02-20T21:52:16.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1493704704 bytes 2026-02-20T21:52:16.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1548678144 bytes 2026-02-20T21:52:16.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1589716992 bytes 2026-02-20T21:52:16.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1597113344 bytes 2026-02-20T21:52:16.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1632764928 bytes 2026-02-20T21:52:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1679364096 bytes 2026-02-20T21:52:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1691277312 bytes 2026-02-20T21:52:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1708071936 bytes 2026-02-20T21:52:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1771272192 bytes 2026-02-20T21:52:16.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1854970880 bytes 2026-02-20T21:52:16.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1925877760 bytes 2026-02-20T21:52:16.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1928264704 bytes 2026-02-20T21:52:16.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1950916608 bytes 2026-02-20T21:52:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1976491008 bytes 2026-02-20T21:52:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2021066752 bytes 2026-02-20T21:52:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2052588544 bytes 2026-02-20T21:52:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2111318016 bytes 2026-02-20T21:52:17.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2162667520 bytes 2026-02-20T21:52:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2176773120 bytes 2026-02-20T21:52:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2186224640 bytes 2026-02-20T21:52:17.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2198677504 bytes 2026-02-20T21:52:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2291213312 bytes 2026-02-20T21:52:17.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2369720320 bytes 2026-02-20T21:52:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2419454976 bytes 2026-02-20T21:52:17.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2453468160 bytes 2026-02-20T21:52:17.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2486921216 bytes 2026-02-20T21:52:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2571968512 bytes 2026-02-20T21:52:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2621461504 bytes 2026-02-20T21:52:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2632007680 bytes 2026-02-20T21:52:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2652698624 bytes 2026-02-20T21:52:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2661452800 bytes 2026-02-20T21:52:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2801588224 bytes 2026-02-20T21:52:18.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2856609792 bytes 2026-02-20T21:52:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2883908608 bytes 2026-02-20T21:52:18.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2915903488 bytes 2026-02-20T21:52:18.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3014071296 bytes 2026-02-20T21:52:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3121604608 bytes 2026-02-20T21:52:18.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3144048640 bytes 2026-02-20T21:52:18.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3167761408 bytes 2026-02-20T21:52:18.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3180370944 bytes 2026-02-20T21:52:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3266198528 bytes 2026-02-20T21:52:18.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3301253120 bytes 2026-02-20T21:52:18.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3306530816 bytes 2026-02-20T21:52:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3334661120 bytes 2026-02-20T21:52:18.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3386686464 bytes 2026-02-20T21:52:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3401098240 bytes 2026-02-20T21:52:18.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3443815424 bytes 2026-02-20T21:52:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3494093824 bytes 2026-02-20T21:52:18.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3554438144 bytes 2026-02-20T21:52:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3599125504 bytes 2026-02-20T21:52:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3628665856 bytes 2026-02-20T21:52:18.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3694643200 bytes 2026-02-20T21:52:18.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3714077696 bytes 2026-02-20T21:52:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3756952576 bytes 2026-02-20T21:52:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3786114048 bytes 2026-02-20T21:52:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3813990400 bytes 2026-02-20T21:52:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3904077824 bytes 2026-02-20T21:52:19.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3925931008 bytes 2026-02-20T21:52:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3974855680 bytes 2026-02-20T21:52:19.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4040754176 bytes 2026-02-20T21:52:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4082696192 bytes 2026-02-20T21:52:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4110172160 bytes 2026-02-20T21:52:19.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4139679744 bytes 2026-02-20T21:52:19.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4159035392 bytes 2026-02-20T21:52:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:52:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:52:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:52:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:52:19.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_27 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 snapshot 128M 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:52:19.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_27 128M 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:52:19.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:52:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:19.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_27 2026-02-20T21:52:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:52:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:52:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_27 snapshot 2026-02-20T21:52:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:19.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T21:52:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_27 snapshot 2026-02-20T21:52:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:52:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_27 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_27': 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13b983746b8d 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13b983746b8d 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:19 2026 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:19 2026 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:19 2026 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:52:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:52:20.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 1b8e35e9-ec45-44f4-a0d7-dbb42eabaaa7 2026-02-20T21:52:20.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:52:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:52:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_27 2026-02-20T21:52:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T21:52:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:52:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=12 2026-02-20T21:52:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:52:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:52:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 69879808 bytes 2026-02-20T21:52:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 130783232 bytes 2026-02-20T21:52:20.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 133406720 bytes 2026-02-20T21:52:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 185340928 bytes 2026-02-20T21:52:20.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 185973760 bytes 2026-02-20T21:52:20.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 244989952 bytes 2026-02-20T21:52:20.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 320899072 bytes 2026-02-20T21:52:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 398223360 bytes 2026-02-20T21:52:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 465473536 bytes 2026-02-20T21:52:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 571699200 bytes 2026-02-20T21:52:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 600503296 bytes 2026-02-20T21:52:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 679958528 bytes 2026-02-20T21:52:21.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 709734400 bytes 2026-02-20T21:52:21.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 778919936 bytes 2026-02-20T21:52:21.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 805264384 bytes 2026-02-20T21:52:21.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 836587520 bytes 2026-02-20T21:52:21.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 903678976 bytes 2026-02-20T21:52:21.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 918789120 bytes 2026-02-20T21:52:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 952196096 bytes 2026-02-20T21:52:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 978036736 bytes 2026-02-20T21:52:21.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1054169088 bytes 2026-02-20T21:52:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1054788608 bytes 2026-02-20T21:52:21.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1074953216 bytes 2026-02-20T21:52:21.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1146241024 bytes 2026-02-20T21:52:21.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1146871808 bytes 2026-02-20T21:52:21.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1169366016 bytes 2026-02-20T21:52:21.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1186043904 bytes 2026-02-20T21:52:21.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1259574272 bytes 2026-02-20T21:52:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1302963200 bytes 2026-02-20T21:52:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1356187648 bytes 2026-02-20T21:52:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1361018880 bytes 2026-02-20T21:52:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1381676032 bytes 2026-02-20T21:52:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1478282240 bytes 2026-02-20T21:52:22.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1493846016 bytes 2026-02-20T21:52:22.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1542169600 bytes 2026-02-20T21:52:22.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1569684480 bytes 2026-02-20T21:52:22.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1697189888 bytes 2026-02-20T21:52:22.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1706606592 bytes 2026-02-20T21:52:22.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1753338880 bytes 2026-02-20T21:52:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1777599488 bytes 2026-02-20T21:52:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1830769664 bytes 2026-02-20T21:52:22.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1903784960 bytes 2026-02-20T21:52:22.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1987058688 bytes 2026-02-20T21:52:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1995734016 bytes 2026-02-20T21:52:22.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2046753792 bytes 2026-02-20T21:52:22.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2057460736 bytes 2026-02-20T21:52:22.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2104707072 bytes 2026-02-20T21:52:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2108280832 bytes 2026-02-20T21:52:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2157392896 bytes 2026-02-20T21:52:23.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2219420672 bytes 2026-02-20T21:52:23.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2243595264 bytes 2026-02-20T21:52:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2304574464 bytes 2026-02-20T21:52:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2407771136 bytes 2026-02-20T21:52:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2480602112 bytes 2026-02-20T21:52:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2502200320 bytes 2026-02-20T21:52:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2544664576 bytes 2026-02-20T21:52:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2597844992 bytes 2026-02-20T21:52:23.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2661087232 bytes 2026-02-20T21:52:23.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2712472576 bytes 2026-02-20T21:52:23.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2791131136 bytes 2026-02-20T21:52:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2793953280 bytes 2026-02-20T21:52:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2806583296 bytes 2026-02-20T21:52:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2857198592 bytes 2026-02-20T21:52:23.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2893165568 bytes 2026-02-20T21:52:23.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2947428352 bytes 2026-02-20T21:52:23.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2977713152 bytes 2026-02-20T21:52:23.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3006559232 bytes 2026-02-20T21:52:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3036095488 bytes 2026-02-20T21:52:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3055393792 bytes 2026-02-20T21:52:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3084416000 bytes 2026-02-20T21:52:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3139921920 bytes 2026-02-20T21:52:24.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3153758208 bytes 2026-02-20T21:52:24.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3158853632 bytes 2026-02-20T21:52:24.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3173945344 bytes 2026-02-20T21:52:24.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3238012928 bytes 2026-02-20T21:52:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3278915584 bytes 2026-02-20T21:52:24.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3345196032 bytes 2026-02-20T21:52:24.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3385961472 bytes 2026-02-20T21:52:24.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3422665728 bytes 2026-02-20T21:52:24.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3469523968 bytes 2026-02-20T21:52:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3520322560 bytes 2026-02-20T21:52:24.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3545633792 bytes 2026-02-20T21:52:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3560736768 bytes 2026-02-20T21:52:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3573512192 bytes 2026-02-20T21:52:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3654661120 bytes 2026-02-20T21:52:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3715381248 bytes 2026-02-20T21:52:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3738998784 bytes 2026-02-20T21:52:24.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3759766528 bytes 2026-02-20T21:52:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3825881088 bytes 2026-02-20T21:52:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3865774080 bytes 2026-02-20T21:52:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3890274304 bytes 2026-02-20T21:52:25.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3970116608 bytes 2026-02-20T21:52:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4026883072 bytes 2026-02-20T21:52:25.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4031731712 bytes 2026-02-20T21:52:25.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4048496640 bytes 2026-02-20T21:52:25.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4130933760 bytes 2026-02-20T21:52:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4246612992 bytes 2026-02-20T21:52:25.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4312101888 bytes 2026-02-20T21:52:25.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4341739520 bytes 2026-02-20T21:52:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4394118144 bytes 2026-02-20T21:52:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4434911232 bytes 2026-02-20T21:52:25.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4523953152 bytes 2026-02-20T21:52:25.632 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7922235927159893) > weight (0.25) 2026-02-20T21:52:25.633 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3131940778733977) > weight (0.25) 2026-02-20T21:52:25.633 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-20T21:52:25.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-20T21:52:25.633 INFO:tasks.rbd_mirror_thrash:waiting for 1.639194412851147 secs before reviving daemons 2026-02-20T21:52:25.634 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T21:52:25.630+0000 7f3dde41d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 26189) UID: 1000 2026-02-20T21:52:25.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4562097152 bytes 2026-02-20T21:52:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4626104320 bytes 2026-02-20T21:52:25.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4640582656 bytes 2026-02-20T21:52:25.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4664181760 bytes 2026-02-20T21:52:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4669443072 bytes 2026-02-20T21:52:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4673740800 bytes 2026-02-20T21:52:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4778558464 bytes 2026-02-20T21:52:25.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4818146304 bytes 2026-02-20T21:52:26.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4845646848 bytes 2026-02-20T21:52:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4890663936 bytes 2026-02-20T21:52:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4935695360 bytes 2026-02-20T21:52:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4971335680 bytes 2026-02-20T21:52:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5048379392 bytes 2026-02-20T21:52:26.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5099170816 bytes 2026-02-20T21:52:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5112414208 bytes 2026-02-20T21:52:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5137607680 bytes 2026-02-20T21:52:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5153202176 bytes 2026-02-20T21:52:26.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5153825792 bytes 2026-02-20T21:52:26.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5173844992 bytes 2026-02-20T21:52:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5188957184 bytes 2026-02-20T21:52:26.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5231106048 bytes 2026-02-20T21:52:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5251131392 bytes 2026-02-20T21:52:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5271350272 bytes 2026-02-20T21:52:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5311233024 bytes 2026-02-20T21:52:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5374482432 bytes 2026-02-20T21:52:26.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5424842752 bytes 2026-02-20T21:52:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5469009920 bytes 2026-02-20T21:52:26.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5475019776 bytes 2026-02-20T21:52:26.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5527161856 bytes 2026-02-20T21:52:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5558077440 bytes 2026-02-20T21:52:27.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5611455488 bytes 2026-02-20T21:52:27.075 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5705032704 bytes 2026-02-20T21:52:27.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5705632768 bytes 2026-02-20T21:52:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5709417472 bytes 2026-02-20T21:52:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5737250816 bytes 2026-02-20T21:52:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5791523840 bytes 2026-02-20T21:52:27.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5801231360 bytes 2026-02-20T21:52:27.273 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-20T21:52:27.273 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:52:27.274 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-20T21:52:27.274 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T21:52:27.274 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T21:52:27.276 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T21:52:27.276 INFO:tasks.rbd_mirror_thrash:waiting for 95.84806570065919 secs before thrashing 2026-02-20T21:52:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5821265920 bytes 2026-02-20T21:52:27.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5835251712 bytes 2026-02-20T21:52:27.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5842585600 bytes 2026-02-20T21:52:27.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5896636416 bytes 2026-02-20T21:52:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5913578496 bytes 2026-02-20T21:52:27.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5985868800 bytes 2026-02-20T21:52:27.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6041096192 bytes 2026-02-20T21:52:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6061733888 bytes 2026-02-20T21:52:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6087586816 bytes 2026-02-20T21:52:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6122894336 bytes 2026-02-20T21:52:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6125701120 bytes 2026-02-20T21:52:27.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6172947456 bytes 2026-02-20T21:52:27.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6231042048 bytes 2026-02-20T21:52:27.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6240260096 bytes 2026-02-20T21:52:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6265915392 bytes 2026-02-20T21:52:27.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6311402496 bytes 2026-02-20T21:52:28.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6353575936 bytes 2026-02-20T21:52:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6380243968 bytes 2026-02-20T21:52:28.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6405647360 bytes 2026-02-20T21:52:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6459186176 bytes 2026-02-20T21:52:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6505358336 bytes 2026-02-20T21:52:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6513600512 bytes 2026-02-20T21:52:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6588295168 bytes 2026-02-20T21:52:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6611747840 bytes 2026-02-20T21:52:28.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6678330368 bytes 2026-02-20T21:52:28.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6681524224 bytes 2026-02-20T21:52:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6703342592 bytes 2026-02-20T21:52:28.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6774301696 bytes 2026-02-20T21:52:28.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6833863680 bytes 2026-02-20T21:52:28.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6910461952 bytes 2026-02-20T21:52:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6949236736 bytes 2026-02-20T21:52:28.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6967357440 bytes 2026-02-20T21:52:28.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7065113600 bytes 2026-02-20T21:52:28.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7119062016 bytes 2026-02-20T21:52:28.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7192767488 bytes 2026-02-20T21:52:28.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7228278784 bytes 2026-02-20T21:52:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7246937088 bytes 2026-02-20T21:52:29.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7365870592 bytes 2026-02-20T21:52:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7404983296 bytes 2026-02-20T21:52:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7441813504 bytes 2026-02-20T21:52:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7476994048 bytes 2026-02-20T21:52:29.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7569293312 bytes 2026-02-20T21:52:29.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7616530432 bytes 2026-02-20T21:52:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7663662080 bytes 2026-02-20T21:52:29.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7705177088 bytes 2026-02-20T21:52:29.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7716501504 bytes 2026-02-20T21:52:29.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7763996672 bytes 2026-02-20T21:52:29.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7800580096 bytes 2026-02-20T21:52:29.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7866707968 bytes 2026-02-20T21:52:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7885465600 bytes 2026-02-20T21:52:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7908472832 bytes 2026-02-20T21:52:29.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7921740800 bytes 2026-02-20T21:52:29.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7984545792 bytes 2026-02-20T21:52:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8033202176 bytes 2026-02-20T21:52:29.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8051699712 bytes 2026-02-20T21:52:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8083247104 bytes 2026-02-20T21:52:30.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8126611456 bytes 2026-02-20T21:52:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8182034432 bytes 2026-02-20T21:52:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8182624256 bytes 2026-02-20T21:52:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8255967232 bytes 2026-02-20T21:52:30.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8319952896 bytes 2026-02-20T21:52:30.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8356924416 bytes 2026-02-20T21:52:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8411439104 bytes 2026-02-20T21:52:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8484526080 bytes 2026-02-20T21:52:30.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8502942720 bytes 2026-02-20T21:52:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8515309568 bytes 2026-02-20T21:52:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8611769344 bytes 2026-02-20T21:52:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8651806720 bytes 2026-02-20T21:52:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8652400640 bytes 2026-02-20T21:52:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8680791040 bytes 2026-02-20T21:52:30.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8747684864 bytes 2026-02-20T21:52:30.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8824115200 bytes 2026-02-20T21:52:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8853007360 bytes 2026-02-20T21:52:30.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8859274240 bytes 2026-02-20T21:52:30.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8946728960 bytes 2026-02-20T21:52:30.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8969843712 bytes 2026-02-20T21:52:30.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9027759104 bytes 2026-02-20T21:52:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9090131968 bytes 2026-02-20T21:52:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9125708800 bytes 2026-02-20T21:52:31.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9199777792 bytes 2026-02-20T21:52:31.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9242277888 bytes 2026-02-20T21:52:31.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9269710848 bytes 2026-02-20T21:52:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9327039488 bytes 2026-02-20T21:52:31.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9361095680 bytes 2026-02-20T21:52:31.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9383790592 bytes 2026-02-20T21:52:31.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9492337664 bytes 2026-02-20T21:52:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9513804800 bytes 2026-02-20T21:52:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9518721024 bytes 2026-02-20T21:52:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9522453504 bytes 2026-02-20T21:52:31.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9582696448 bytes 2026-02-20T21:52:31.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9610754048 bytes 2026-02-20T21:52:31.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9665617920 bytes 2026-02-20T21:52:31.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9705800704 bytes 2026-02-20T21:52:31.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9773874176 bytes 2026-02-20T21:52:31.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9839207424 bytes 2026-02-20T21:52:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9882982400 bytes 2026-02-20T21:52:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 9907897344 bytes 2026-02-20T21:52:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 9956461568 bytes 2026-02-20T21:52:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 10004342784 bytes 2026-02-20T21:52:32.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 10047709184 bytes 2026-02-20T21:52:32.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 10076724224 bytes 2026-02-20T21:52:32.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 10117730304 bytes 2026-02-20T21:52:32.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10136291328 bytes 2026-02-20T21:52:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 10152899584 bytes 2026-02-20T21:52:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 10197018624 bytes 2026-02-20T21:52:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10217296896 bytes 2026-02-20T21:52:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10246839296 bytes 2026-02-20T21:52:32.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10279750656 bytes 2026-02-20T21:52:32.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10296929280 bytes 2026-02-20T21:52:32.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 10332897280 bytes 2026-02-20T21:52:32.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_28 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 snapshot 128M 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_28 128M 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T21:52:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:52:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:52:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:52:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_28 2026-02-20T21:52:32.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:52:32.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:32.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:52:32.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_28 snapshot 2026-02-20T21:52:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T21:52:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_28 snapshot 2026-02-20T21:52:33.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:52:33.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_28 2026-02-20T21:52:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_28': 2026-02-20T21:52:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:52:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:52:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:52:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13ce60afd6b8 2026-02-20T21:52:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13ce60afd6b8 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:32 2026 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:32 2026 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:32 2026 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: ab5b8a6b-3393-4121-92ba-22af9b8d3ce4 2026-02-20T21:52:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:52:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:52:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_28 2026-02-20T21:52:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T21:52:33.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:52:33.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=5 2026-02-20T21:52:33.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:52:33.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:52:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 607232 bytes 2026-02-20T21:52:33.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 69652480 bytes 2026-02-20T21:52:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 97833984 bytes 2026-02-20T21:52:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 149837824 bytes 2026-02-20T21:52:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 198792192 bytes 2026-02-20T21:52:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 235269120 bytes 2026-02-20T21:52:33.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 283761664 bytes 2026-02-20T21:52:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 344255488 bytes 2026-02-20T21:52:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 361066496 bytes 2026-02-20T21:52:34.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 429070336 bytes 2026-02-20T21:52:34.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 469877760 bytes 2026-02-20T21:52:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 518495232 bytes 2026-02-20T21:52:34.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 551569408 bytes 2026-02-20T21:52:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 587926528 bytes 2026-02-20T21:52:34.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 643763200 bytes 2026-02-20T21:52:34.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 662829056 bytes 2026-02-20T21:52:34.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 712559616 bytes 2026-02-20T21:52:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 786379776 bytes 2026-02-20T21:52:34.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 827644928 bytes 2026-02-20T21:52:34.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 875507712 bytes 2026-02-20T21:52:34.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 909186048 bytes 2026-02-20T21:52:34.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 919495680 bytes 2026-02-20T21:52:34.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 940098560 bytes 2026-02-20T21:52:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1015398400 bytes 2026-02-20T21:52:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1062334464 bytes 2026-02-20T21:52:34.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1135027200 bytes 2026-02-20T21:52:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1163754496 bytes 2026-02-20T21:52:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1238761472 bytes 2026-02-20T21:52:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1241118720 bytes 2026-02-20T21:52:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1347132416 bytes 2026-02-20T21:52:35.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1371759616 bytes 2026-02-20T21:52:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1402656768 bytes 2026-02-20T21:52:35.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1404765184 bytes 2026-02-20T21:52:35.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1430623232 bytes 2026-02-20T21:52:35.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1467502592 bytes 2026-02-20T21:52:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1474110464 bytes 2026-02-20T21:52:35.326 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:52:35.322+0000 7fa5e2658640 -1 reset not still connected to 0x55b92c1084e0 2026-02-20T21:52:35.327 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:52:35.322+0000 7fa5e2658640 -1 reset not still connected to 0x55b92c1096c0 2026-02-20T21:52:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1499597824 bytes 2026-02-20T21:52:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1542279168 bytes 2026-02-20T21:52:35.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1562074112 bytes 2026-02-20T21:52:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1586292736 bytes 2026-02-20T21:52:35.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1600664576 bytes 2026-02-20T21:52:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1678540800 bytes 2026-02-20T21:52:35.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1712440320 bytes 2026-02-20T21:52:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1756684288 bytes 2026-02-20T21:52:35.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1797277696 bytes 2026-02-20T21:52:35.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1807207424 bytes 2026-02-20T21:52:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1828629504 bytes 2026-02-20T21:52:35.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1900925952 bytes 2026-02-20T21:52:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1952441344 bytes 2026-02-20T21:52:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1984197632 bytes 2026-02-20T21:52:36.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2022168576 bytes 2026-02-20T21:52:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2070934528 bytes 2026-02-20T21:52:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2120244224 bytes 2026-02-20T21:52:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2137958400 bytes 2026-02-20T21:52:36.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2161769472 bytes 2026-02-20T21:52:36.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2201970688 bytes 2026-02-20T21:52:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2290378752 bytes 2026-02-20T21:52:36.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2336975872 bytes 2026-02-20T21:52:36.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2383994880 bytes 2026-02-20T21:52:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2429986816 bytes 2026-02-20T21:52:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2457291776 bytes 2026-02-20T21:52:36.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2525153280 bytes 2026-02-20T21:52:36.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2548182016 bytes 2026-02-20T21:52:36.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2601610240 bytes 2026-02-20T21:52:36.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2627796992 bytes 2026-02-20T21:52:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2653351936 bytes 2026-02-20T21:52:36.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2701328384 bytes 2026-02-20T21:52:36.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2756619264 bytes 2026-02-20T21:52:36.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2805579776 bytes 2026-02-20T21:52:36.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2845333504 bytes 2026-02-20T21:52:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2866199552 bytes 2026-02-20T21:52:37.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2876493824 bytes 2026-02-20T21:52:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2906003456 bytes 2026-02-20T21:52:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2943337472 bytes 2026-02-20T21:52:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2995437568 bytes 2026-02-20T21:52:37.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3080110080 bytes 2026-02-20T21:52:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3095275520 bytes 2026-02-20T21:52:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3129819136 bytes 2026-02-20T21:52:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3193497600 bytes 2026-02-20T21:52:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3220342784 bytes 2026-02-20T21:52:37.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3245529088 bytes 2026-02-20T21:52:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3315664896 bytes 2026-02-20T21:52:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3351482368 bytes 2026-02-20T21:52:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3432894464 bytes 2026-02-20T21:52:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3433517056 bytes 2026-02-20T21:52:37.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3498425344 bytes 2026-02-20T21:52:37.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3535529984 bytes 2026-02-20T21:52:37.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3636251648 bytes 2026-02-20T21:52:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3663971328 bytes 2026-02-20T21:52:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3736296448 bytes 2026-02-20T21:52:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3758812160 bytes 2026-02-20T21:52:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3834205184 bytes 2026-02-20T21:52:38.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3865579520 bytes 2026-02-20T21:52:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3907181568 bytes 2026-02-20T21:52:38.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3929926656 bytes 2026-02-20T21:52:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3962928128 bytes 2026-02-20T21:52:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3970051072 bytes 2026-02-20T21:52:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3995324416 bytes 2026-02-20T21:52:38.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4019397632 bytes 2026-02-20T21:52:38.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4097191936 bytes 2026-02-20T21:52:38.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4153714688 bytes 2026-02-20T21:52:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:52:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:52:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:52:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:52:38.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_29 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 snapshot 128M 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_29 128M 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:38.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_29 2026-02-20T21:52:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:52:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:52:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_29 snapshot 2026-02-20T21:52:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T21:52:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_29 snapshot 2026-02-20T21:52:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:52:39.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_29 2026-02-20T21:52:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_29': 2026-02-20T21:52:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13da342636fc 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13da342636fc 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:38 2026 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:38 2026 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:38 2026 2026-02-20T21:52:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:52:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:52:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 2d03f9ea-eefd-44d9-b8bd-e230886e2bc8 2026-02-20T21:52:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:52:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:52:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_29 2026-02-20T21:52:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T21:52:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:52:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=5 2026-02-20T21:52:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:52:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:52:39.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 45178880 bytes 2026-02-20T21:52:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 117370880 bytes 2026-02-20T21:52:39.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 137802752 bytes 2026-02-20T21:52:39.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 174224384 bytes 2026-02-20T21:52:39.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 251761664 bytes 2026-02-20T21:52:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 271080448 bytes 2026-02-20T21:52:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 273153024 bytes 2026-02-20T21:52:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 322340864 bytes 2026-02-20T21:52:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 373782528 bytes 2026-02-20T21:52:40.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 428989440 bytes 2026-02-20T21:52:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 450442240 bytes 2026-02-20T21:52:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 520716288 bytes 2026-02-20T21:52:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 545392640 bytes 2026-02-20T21:52:40.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 579264512 bytes 2026-02-20T21:52:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 613252096 bytes 2026-02-20T21:52:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 672075776 bytes 2026-02-20T21:52:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 735596544 bytes 2026-02-20T21:52:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 780729344 bytes 2026-02-20T21:52:40.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 830007296 bytes 2026-02-20T21:52:40.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 902207488 bytes 2026-02-20T21:52:40.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 974991360 bytes 2026-02-20T21:52:40.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1024572416 bytes 2026-02-20T21:52:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1057971200 bytes 2026-02-20T21:52:40.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1078728704 bytes 2026-02-20T21:52:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1111717888 bytes 2026-02-20T21:52:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1132267520 bytes 2026-02-20T21:52:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1183022080 bytes 2026-02-20T21:52:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1221465088 bytes 2026-02-20T21:52:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1238323200 bytes 2026-02-20T21:52:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1270317056 bytes 2026-02-20T21:52:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1361857536 bytes 2026-02-20T21:52:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1396480000 bytes 2026-02-20T21:52:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1464429568 bytes 2026-02-20T21:52:41.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1465036800 bytes 2026-02-20T21:52:41.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1489897472 bytes 2026-02-20T21:52:41.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1568231424 bytes 2026-02-20T21:52:41.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1617553408 bytes 2026-02-20T21:52:41.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1637036032 bytes 2026-02-20T21:52:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1697036288 bytes 2026-02-20T21:52:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1771393024 bytes 2026-02-20T21:52:41.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1771993088 bytes 2026-02-20T21:52:41.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1827155968 bytes 2026-02-20T21:52:41.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1891222528 bytes 2026-02-20T21:52:41.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1910757376 bytes 2026-02-20T21:52:41.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1958418432 bytes 2026-02-20T21:52:41.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2063136768 bytes 2026-02-20T21:52:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2086169600 bytes 2026-02-20T21:52:42.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2108630016 bytes 2026-02-20T21:52:42.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2127682560 bytes 2026-02-20T21:52:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2157948928 bytes 2026-02-20T21:52:42.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2185024512 bytes 2026-02-20T21:52:42.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2223821824 bytes 2026-02-20T21:52:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2306630656 bytes 2026-02-20T21:52:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2307580928 bytes 2026-02-20T21:52:42.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2325398528 bytes 2026-02-20T21:52:42.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2348070912 bytes 2026-02-20T21:52:42.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2376531968 bytes 2026-02-20T21:52:42.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2456756224 bytes 2026-02-20T21:52:42.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2544369664 bytes 2026-02-20T21:52:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2594540544 bytes 2026-02-20T21:52:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2644627456 bytes 2026-02-20T21:52:42.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2645545984 bytes 2026-02-20T21:52:42.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2698114048 bytes 2026-02-20T21:52:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2734590976 bytes 2026-02-20T21:52:42.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2803836928 bytes 2026-02-20T21:52:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2869629952 bytes 2026-02-20T21:52:42.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2903986176 bytes 2026-02-20T21:52:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3011504128 bytes 2026-02-20T21:52:43.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3012116480 bytes 2026-02-20T21:52:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3026999296 bytes 2026-02-20T21:52:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3051507712 bytes 2026-02-20T21:52:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3087967232 bytes 2026-02-20T21:52:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3148302336 bytes 2026-02-20T21:52:43.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3169469440 bytes 2026-02-20T21:52:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3184423936 bytes 2026-02-20T21:52:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3228706816 bytes 2026-02-20T21:52:43.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3268331520 bytes 2026-02-20T21:52:43.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3306822656 bytes 2026-02-20T21:52:43.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3329528832 bytes 2026-02-20T21:52:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3369016320 bytes 2026-02-20T21:52:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3370008576 bytes 2026-02-20T21:52:43.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3393341440 bytes 2026-02-20T21:52:43.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3459947520 bytes 2026-02-20T21:52:43.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3585890304 bytes 2026-02-20T21:52:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3597720576 bytes 2026-02-20T21:52:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3629486080 bytes 2026-02-20T21:52:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3675307008 bytes 2026-02-20T21:52:44.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3687921664 bytes 2026-02-20T21:52:44.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3720876032 bytes 2026-02-20T21:52:44.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3749699584 bytes 2026-02-20T21:52:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3773180928 bytes 2026-02-20T21:52:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3822157824 bytes 2026-02-20T21:52:44.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3884490752 bytes 2026-02-20T21:52:44.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3909792768 bytes 2026-02-20T21:52:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3940822016 bytes 2026-02-20T21:52:44.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4003589120 bytes 2026-02-20T21:52:44.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4047941632 bytes 2026-02-20T21:52:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:52:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_30 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 snapshot 128M 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T21:52:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_30 128M 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:52:44.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:52:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:44.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_30 2026-02-20T21:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_30 snapshot 2026-02-20T21:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T21:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_30 snapshot 2026-02-20T21:52:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:52:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_30 2026-02-20T21:52:45.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_30': 2026-02-20T21:52:45.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13e6fbd4427a 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13e6fbd4427a 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:44 2026 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:44 2026 2026-02-20T21:52:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:44 2026 2026-02-20T21:52:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:52:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:52:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: d649a580-cc10-448a-bdd7-2ee6987b9a69 2026-02-20T21:52:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:52:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:52:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_30 2026-02-20T21:52:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T21:52:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:52:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=7 2026-02-20T21:52:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:52:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:52:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 25172992 bytes 2026-02-20T21:52:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 29656064 bytes 2026-02-20T21:52:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 40729600 bytes 2026-02-20T21:52:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 96381952 bytes 2026-02-20T21:52:45.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 100302848 bytes 2026-02-20T21:52:45.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 135173120 bytes 2026-02-20T21:52:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 143815680 bytes 2026-02-20T21:52:45.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 183675904 bytes 2026-02-20T21:52:45.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 204040192 bytes 2026-02-20T21:52:46.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 271516672 bytes 2026-02-20T21:52:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 293395456 bytes 2026-02-20T21:52:46.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 344714240 bytes 2026-02-20T21:52:46.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 424274944 bytes 2026-02-20T21:52:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 473990144 bytes 2026-02-20T21:52:46.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 534971392 bytes 2026-02-20T21:52:46.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 551667712 bytes 2026-02-20T21:52:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 566634496 bytes 2026-02-20T21:52:46.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 610881536 bytes 2026-02-20T21:52:46.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 662322176 bytes 2026-02-20T21:52:46.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 675640320 bytes 2026-02-20T21:52:46.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 676259840 bytes 2026-02-20T21:52:46.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 690628608 bytes 2026-02-20T21:52:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 724801536 bytes 2026-02-20T21:52:46.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 725400576 bytes 2026-02-20T21:52:46.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 754450432 bytes 2026-02-20T21:52:46.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 797146112 bytes 2026-02-20T21:52:46.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 848801792 bytes 2026-02-20T21:52:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 934221824 bytes 2026-02-20T21:52:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 982512640 bytes 2026-02-20T21:52:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1071234048 bytes 2026-02-20T21:52:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1106026496 bytes 2026-02-20T21:52:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1145662464 bytes 2026-02-20T21:52:47.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1224879104 bytes 2026-02-20T21:52:47.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1225515008 bytes 2026-02-20T21:52:47.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1262618624 bytes 2026-02-20T21:52:47.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1298656256 bytes 2026-02-20T21:52:47.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1382307840 bytes 2026-02-20T21:52:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1437233152 bytes 2026-02-20T21:52:47.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1476124672 bytes 2026-02-20T21:52:47.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1529371648 bytes 2026-02-20T21:52:47.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1570330624 bytes 2026-02-20T21:52:47.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1584605184 bytes 2026-02-20T21:52:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1722781696 bytes 2026-02-20T21:52:47.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1756049408 bytes 2026-02-20T21:52:47.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1795272704 bytes 2026-02-20T21:52:47.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1855783936 bytes 2026-02-20T21:52:47.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1877825536 bytes 2026-02-20T21:52:47.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1914976256 bytes 2026-02-20T21:52:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1936917504 bytes 2026-02-20T21:52:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1985289216 bytes 2026-02-20T21:52:48.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2012058624 bytes 2026-02-20T21:52:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2032738304 bytes 2026-02-20T21:52:48.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2058024960 bytes 2026-02-20T21:52:48.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2058632192 bytes 2026-02-20T21:52:48.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2088235008 bytes 2026-02-20T21:52:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2154947584 bytes 2026-02-20T21:52:48.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2213991424 bytes 2026-02-20T21:52:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2269799424 bytes 2026-02-20T21:52:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2350173184 bytes 2026-02-20T21:52:48.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2370397184 bytes 2026-02-20T21:52:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2393734144 bytes 2026-02-20T21:52:48.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2430581760 bytes 2026-02-20T21:52:48.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2495704064 bytes 2026-02-20T21:52:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2565593088 bytes 2026-02-20T21:52:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2618663936 bytes 2026-02-20T21:52:48.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2624659456 bytes 2026-02-20T21:52:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2674314240 bytes 2026-02-20T21:52:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2687637504 bytes 2026-02-20T21:52:48.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2728220672 bytes 2026-02-20T21:52:49.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2794139648 bytes 2026-02-20T21:52:49.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2827521024 bytes 2026-02-20T21:52:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2864909312 bytes 2026-02-20T21:52:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2904664064 bytes 2026-02-20T21:52:49.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2941245440 bytes 2026-02-20T21:52:49.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2941864960 bytes 2026-02-20T21:52:49.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 2973568000 bytes 2026-02-20T21:52:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 2992347136 bytes 2026-02-20T21:52:49.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3034813440 bytes 2026-02-20T21:52:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3107416064 bytes 2026-02-20T21:52:49.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3109353472 bytes 2026-02-20T21:52:49.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3142936576 bytes 2026-02-20T21:52:49.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3157689344 bytes 2026-02-20T21:52:49.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3165943808 bytes 2026-02-20T21:52:49.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3212982272 bytes 2026-02-20T21:52:49.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3238269952 bytes 2026-02-20T21:52:49.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3253638144 bytes 2026-02-20T21:52:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3305547776 bytes 2026-02-20T21:52:49.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3321092096 bytes 2026-02-20T21:52:49.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3408784384 bytes 2026-02-20T21:52:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3426906112 bytes 2026-02-20T21:52:49.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3468776448 bytes 2026-02-20T21:52:49.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3551440896 bytes 2026-02-20T21:52:50.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3610847232 bytes 2026-02-20T21:52:50.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3692128256 bytes 2026-02-20T21:52:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3707046912 bytes 2026-02-20T21:52:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3718238208 bytes 2026-02-20T21:52:50.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3755891712 bytes 2026-02-20T21:52:50.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3789494272 bytes 2026-02-20T21:52:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3808711680 bytes 2026-02-20T21:52:50.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3878767616 bytes 2026-02-20T21:52:50.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 3950928896 bytes 2026-02-20T21:52:50.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 3954116608 bytes 2026-02-20T21:52:50.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 3975880704 bytes 2026-02-20T21:52:50.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4008624128 bytes 2026-02-20T21:52:50.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4020721664 bytes 2026-02-20T21:52:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4052188160 bytes 2026-02-20T21:52:50.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4070858752 bytes 2026-02-20T21:52:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4100316160 bytes 2026-02-20T21:52:50.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4168632320 bytes 2026-02-20T21:52:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4204643328 bytes 2026-02-20T21:52:50.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4205267968 bytes 2026-02-20T21:52:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4297698304 bytes 2026-02-20T21:52:50.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4351736832 bytes 2026-02-20T21:52:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4387781632 bytes 2026-02-20T21:52:51.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4460024832 bytes 2026-02-20T21:52:51.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4526602240 bytes 2026-02-20T21:52:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4562487296 bytes 2026-02-20T21:52:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4563078144 bytes 2026-02-20T21:52:51.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4600611840 bytes 2026-02-20T21:52:51.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4616450048 bytes 2026-02-20T21:52:51.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4643642368 bytes 2026-02-20T21:52:51.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4669580288 bytes 2026-02-20T21:52:51.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4800498688 bytes 2026-02-20T21:52:51.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4849801216 bytes 2026-02-20T21:52:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4891773952 bytes 2026-02-20T21:52:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 4913029120 bytes 2026-02-20T21:52:51.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 4930163712 bytes 2026-02-20T21:52:51.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5000051712 bytes 2026-02-20T21:52:51.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5030029312 bytes 2026-02-20T21:52:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5030653952 bytes 2026-02-20T21:52:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5082465280 bytes 2026-02-20T21:52:51.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5096470528 bytes 2026-02-20T21:52:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5097223168 bytes 2026-02-20T21:52:51.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5119130624 bytes 2026-02-20T21:52:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5136827392 bytes 2026-02-20T21:52:51.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5157278720 bytes 2026-02-20T21:52:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5248078848 bytes 2026-02-20T21:52:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5306109952 bytes 2026-02-20T21:52:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5377371136 bytes 2026-02-20T21:52:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5394221056 bytes 2026-02-20T21:52:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5415411712 bytes 2026-02-20T21:52:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5444188160 bytes 2026-02-20T21:52:52.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5456918528 bytes 2026-02-20T21:52:52.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5538521088 bytes 2026-02-20T21:52:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5578103808 bytes 2026-02-20T21:52:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5592651776 bytes 2026-02-20T21:52:52.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5624997888 bytes 2026-02-20T21:52:52.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5626413056 bytes 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_31 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 snapshot 128M 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_31 128M 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:52.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T21:52:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:52:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:52:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:52:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:52:52.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_31 2026-02-20T21:52:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:52:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:52:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:52:52.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_31 snapshot 2026-02-20T21:52:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T21:52:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:52:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_31 snapshot 2026-02-20T21:52:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:52:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_31 2026-02-20T21:52:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_31': 2026-02-20T21:52:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:52:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13f2514ef61f 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13f2514ef61f 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:52 2026 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:52 2026 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:52 2026 2026-02-20T21:52:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:52:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:52:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 28720fa9-2640-4d05-b3cb-cf921814d23d 2026-02-20T21:52:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:52:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:52:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_31 2026-02-20T21:52:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:52:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:52:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T21:52:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:52:53.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=14 2026-02-20T21:52:53.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:52:53.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:52:53.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 75407360 bytes 2026-02-20T21:52:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 161120256 bytes 2026-02-20T21:52:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 200335360 bytes 2026-02-20T21:52:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 242231296 bytes 2026-02-20T21:52:53.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 269124608 bytes 2026-02-20T21:52:53.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 319319040 bytes 2026-02-20T21:52:54.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 360895488 bytes 2026-02-20T21:52:54.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 393463808 bytes 2026-02-20T21:52:54.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 459025408 bytes 2026-02-20T21:52:54.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 479995904 bytes 2026-02-20T21:52:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 514600960 bytes 2026-02-20T21:52:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 518076416 bytes 2026-02-20T21:52:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 559886336 bytes 2026-02-20T21:52:54.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 652235776 bytes 2026-02-20T21:52:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 702331904 bytes 2026-02-20T21:52:54.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 803277824 bytes 2026-02-20T21:52:54.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 832684032 bytes 2026-02-20T21:52:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 870633472 bytes 2026-02-20T21:52:54.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 897653760 bytes 2026-02-20T21:52:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1001060352 bytes 2026-02-20T21:52:54.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1004289024 bytes 2026-02-20T21:52:54.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1045016576 bytes 2026-02-20T21:52:55.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1113629696 bytes 2026-02-20T21:52:55.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1143568384 bytes 2026-02-20T21:52:55.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1168700416 bytes 2026-02-20T21:52:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1169314816 bytes 2026-02-20T21:52:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1175985152 bytes 2026-02-20T21:52:55.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1193331712 bytes 2026-02-20T21:52:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1226251264 bytes 2026-02-20T21:52:55.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1227680768 bytes 2026-02-20T21:52:55.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1304436736 bytes 2026-02-20T21:52:55.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1317793792 bytes 2026-02-20T21:52:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1346318336 bytes 2026-02-20T21:52:55.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1391583232 bytes 2026-02-20T21:52:55.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1442541568 bytes 2026-02-20T21:52:55.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1447021568 bytes 2026-02-20T21:52:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1447640064 bytes 2026-02-20T21:52:55.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1528201216 bytes 2026-02-20T21:52:55.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1574886400 bytes 2026-02-20T21:52:55.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1586767872 bytes 2026-02-20T21:52:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1611132928 bytes 2026-02-20T21:52:55.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1615990784 bytes 2026-02-20T21:52:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1669308416 bytes 2026-02-20T21:52:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1703223296 bytes 2026-02-20T21:52:55.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1755019264 bytes 2026-02-20T21:52:56.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1819194368 bytes 2026-02-20T21:52:56.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1921671168 bytes 2026-02-20T21:52:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1951076352 bytes 2026-02-20T21:52:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1991397376 bytes 2026-02-20T21:52:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1993105408 bytes 2026-02-20T21:52:56.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2017346560 bytes 2026-02-20T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2110496768 bytes 2026-02-20T21:52:56.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2143226880 bytes 2026-02-20T21:52:56.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2160530432 bytes 2026-02-20T21:52:56.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2164798464 bytes 2026-02-20T21:52:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2232690688 bytes 2026-02-20T21:52:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2314899456 bytes 2026-02-20T21:52:56.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2343736320 bytes 2026-02-20T21:52:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2382112768 bytes 2026-02-20T21:52:56.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2405228544 bytes 2026-02-20T21:52:56.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2428680192 bytes 2026-02-20T21:52:56.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2451613696 bytes 2026-02-20T21:52:56.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2503136256 bytes 2026-02-20T21:52:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2540943360 bytes 2026-02-20T21:52:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2541566976 bytes 2026-02-20T21:52:57.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2647014400 bytes 2026-02-20T21:52:57.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2665485312 bytes 2026-02-20T21:52:57.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2705889280 bytes 2026-02-20T21:52:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2747977728 bytes 2026-02-20T21:52:57.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2800800768 bytes 2026-02-20T21:52:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2808041472 bytes 2026-02-20T21:52:57.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2858714112 bytes 2026-02-20T21:52:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2888485888 bytes 2026-02-20T21:52:57.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2935333888 bytes 2026-02-20T21:52:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2967840768 bytes 2026-02-20T21:52:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3071531008 bytes 2026-02-20T21:52:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3155249152 bytes 2026-02-20T21:52:57.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3176031232 bytes 2026-02-20T21:52:57.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3200971776 bytes 2026-02-20T21:52:57.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3260850176 bytes 2026-02-20T21:52:57.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3353702400 bytes 2026-02-20T21:52:58.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3379379200 bytes 2026-02-20T21:52:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3446037504 bytes 2026-02-20T21:52:58.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3514801152 bytes 2026-02-20T21:52:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3603387392 bytes 2026-02-20T21:52:58.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3655057408 bytes 2026-02-20T21:52:58.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3672882176 bytes 2026-02-20T21:52:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3684116480 bytes 2026-02-20T21:52:58.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3739784192 bytes 2026-02-20T21:52:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3785176064 bytes 2026-02-20T21:52:58.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3850277888 bytes 2026-02-20T21:52:58.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3905010688 bytes 2026-02-20T21:52:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3918752768 bytes 2026-02-20T21:52:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3965597696 bytes 2026-02-20T21:52:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4018850816 bytes 2026-02-20T21:52:58.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4074345472 bytes 2026-02-20T21:52:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4100940800 bytes 2026-02-20T21:52:58.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4151629824 bytes 2026-02-20T21:52:58.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4165356544 bytes 2026-02-20T21:52:58.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4216652800 bytes 2026-02-20T21:52:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4241722368 bytes 2026-02-20T21:52:59.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4249082880 bytes 2026-02-20T21:52:59.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4249705472 bytes 2026-02-20T21:52:59.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4313240576 bytes 2026-02-20T21:52:59.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4415947776 bytes 2026-02-20T21:52:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4467180544 bytes 2026-02-20T21:52:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4512710656 bytes 2026-02-20T21:52:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4579393536 bytes 2026-02-20T21:52:59.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4604643328 bytes 2026-02-20T21:52:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4672210944 bytes 2026-02-20T21:52:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4693847040 bytes 2026-02-20T21:52:59.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4763016192 bytes 2026-02-20T21:52:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4818588672 bytes 2026-02-20T21:52:59.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4858065920 bytes 2026-02-20T21:52:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4867475456 bytes 2026-02-20T21:52:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4918474752 bytes 2026-02-20T21:52:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4927174656 bytes 2026-02-20T21:52:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4984292352 bytes 2026-02-20T21:52:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5022012416 bytes 2026-02-20T21:52:59.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5077837824 bytes 2026-02-20T21:53:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5137862656 bytes 2026-02-20T21:53:00.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5216026624 bytes 2026-02-20T21:53:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5297639424 bytes 2026-02-20T21:53:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5365655552 bytes 2026-02-20T21:53:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5383999488 bytes 2026-02-20T21:53:00.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5406083072 bytes 2026-02-20T21:53:00.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5412007936 bytes 2026-02-20T21:53:00.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5439912960 bytes 2026-02-20T21:53:00.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5449692160 bytes 2026-02-20T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5469204480 bytes 2026-02-20T21:53:00.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5495653376 bytes 2026-02-20T21:53:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5534048256 bytes 2026-02-20T21:53:00.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5546786816 bytes 2026-02-20T21:53:00.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5579881472 bytes 2026-02-20T21:53:00.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5639360512 bytes 2026-02-20T21:53:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5674921984 bytes 2026-02-20T21:53:00.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5715504128 bytes 2026-02-20T21:53:00.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5752107008 bytes 2026-02-20T21:53:00.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5801408512 bytes 2026-02-20T21:53:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5815609344 bytes 2026-02-20T21:53:00.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5816228864 bytes 2026-02-20T21:53:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5857559552 bytes 2026-02-20T21:53:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5939026944 bytes 2026-02-20T21:53:01.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5968825344 bytes 2026-02-20T21:53:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6035514368 bytes 2026-02-20T21:53:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6076262400 bytes 2026-02-20T21:53:01.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6102322176 bytes 2026-02-20T21:53:01.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6140457984 bytes 2026-02-20T21:53:01.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6154668032 bytes 2026-02-20T21:53:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6250528768 bytes 2026-02-20T21:53:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6292249600 bytes 2026-02-20T21:53:01.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6366535680 bytes 2026-02-20T21:53:01.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6476442624 bytes 2026-02-20T21:53:01.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6611222528 bytes 2026-02-20T21:53:01.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6661905408 bytes 2026-02-20T21:53:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6687503360 bytes 2026-02-20T21:53:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6758473728 bytes 2026-02-20T21:53:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6780521472 bytes 2026-02-20T21:53:02.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6781473792 bytes 2026-02-20T21:53:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6822137856 bytes 2026-02-20T21:53:02.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6938943488 bytes 2026-02-20T21:53:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7002487808 bytes 2026-02-20T21:53:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7060465664 bytes 2026-02-20T21:53:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7079770112 bytes 2026-02-20T21:53:02.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7116710912 bytes 2026-02-20T21:53:02.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7162439680 bytes 2026-02-20T21:53:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7216766976 bytes 2026-02-20T21:53:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7279496192 bytes 2026-02-20T21:53:02.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7314450432 bytes 2026-02-20T21:53:02.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7322684416 bytes 2026-02-20T21:53:02.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7331746816 bytes 2026-02-20T21:53:02.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7376281600 bytes 2026-02-20T21:53:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7378242560 bytes 2026-02-20T21:53:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7385847808 bytes 2026-02-20T21:53:02.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7406432256 bytes 2026-02-20T21:53:02.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7414437888 bytes 2026-02-20T21:53:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7457162240 bytes 2026-02-20T21:53:02.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7482820608 bytes 2026-02-20T21:53:03.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7531837440 bytes 2026-02-20T21:53:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7554495488 bytes 2026-02-20T21:53:03.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7622176768 bytes 2026-02-20T21:53:03.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7665385472 bytes 2026-02-20T21:53:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7759755264 bytes 2026-02-20T21:53:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7776480256 bytes 2026-02-20T21:53:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7784914944 bytes 2026-02-20T21:53:03.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7810526208 bytes 2026-02-20T21:53:03.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7875665920 bytes 2026-02-20T21:53:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7898850304 bytes 2026-02-20T21:53:03.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7969524736 bytes 2026-02-20T21:53:03.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8074248192 bytes 2026-02-20T21:53:03.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8141074432 bytes 2026-02-20T21:53:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8217424896 bytes 2026-02-20T21:53:03.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8260600832 bytes 2026-02-20T21:53:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8314014720 bytes 2026-02-20T21:53:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8342124544 bytes 2026-02-20T21:53:03.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8361367552 bytes 2026-02-20T21:53:03.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8423986176 bytes 2026-02-20T21:53:04.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8488315904 bytes 2026-02-20T21:53:04.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8540064768 bytes 2026-02-20T21:53:04.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8586489856 bytes 2026-02-20T21:53:04.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8612056064 bytes 2026-02-20T21:53:04.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8627719168 bytes 2026-02-20T21:53:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8660691968 bytes 2026-02-20T21:53:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8674893824 bytes 2026-02-20T21:53:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8769047552 bytes 2026-02-20T21:53:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8832740352 bytes 2026-02-20T21:53:04.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8849398784 bytes 2026-02-20T21:53:04.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8915933184 bytes 2026-02-20T21:53:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8965033984 bytes 2026-02-20T21:53:04.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8978345984 bytes 2026-02-20T21:53:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9062073344 bytes 2026-02-20T21:53:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9105731584 bytes 2026-02-20T21:53:04.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9145169920 bytes 2026-02-20T21:53:04.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9209980928 bytes 2026-02-20T21:53:04.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9254549504 bytes 2026-02-20T21:53:04.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9276802048 bytes 2026-02-20T21:53:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9298519040 bytes 2026-02-20T21:53:05.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9354481664 bytes 2026-02-20T21:53:05.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9408073728 bytes 2026-02-20T21:53:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9432001536 bytes 2026-02-20T21:53:05.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9457797120 bytes 2026-02-20T21:53:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9461590016 bytes 2026-02-20T21:53:05.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9485369344 bytes 2026-02-20T21:53:05.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9523514368 bytes 2026-02-20T21:53:05.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9587291136 bytes 2026-02-20T21:53:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9631191040 bytes 2026-02-20T21:53:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9657425920 bytes 2026-02-20T21:53:05.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9704004608 bytes 2026-02-20T21:53:05.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9767795712 bytes 2026-02-20T21:53:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9835057152 bytes 2026-02-20T21:53:05.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9836744704 bytes 2026-02-20T21:53:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9916976128 bytes 2026-02-20T21:53:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9980886016 bytes 2026-02-20T21:53:05.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 10011837440 bytes 2026-02-20T21:53:05.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 10037492736 bytes 2026-02-20T21:53:05.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 10060457984 bytes 2026-02-20T21:53:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 10094891008 bytes 2026-02-20T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 10126063616 bytes 2026-02-20T21:53:06.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 10157386752 bytes 2026-02-20T21:53:06.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 10170373120 bytes 2026-02-20T21:53:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 10191210496 bytes 2026-02-20T21:53:06.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10297250816 bytes 2026-02-20T21:53:06.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 10341794816 bytes 2026-02-20T21:53:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 10357376000 bytes 2026-02-20T21:53:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10392639488 bytes 2026-02-20T21:53:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10439304192 bytes 2026-02-20T21:53:06.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10492076032 bytes 2026-02-20T21:53:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10523566080 bytes 2026-02-20T21:53:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 10530131968 bytes 2026-02-20T21:53:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 10579755008 bytes 2026-02-20T21:53:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25100 IOs, 10677605376 bytes 2026-02-20T21:53:06.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25200 IOs, 10721854464 bytes 2026-02-20T21:53:06.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25300 IOs, 10788827136 bytes 2026-02-20T21:53:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25400 IOs, 10789427200 bytes 2026-02-20T21:53:06.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25500 IOs, 10795965440 bytes 2026-02-20T21:53:06.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25600 IOs, 10836603904 bytes 2026-02-20T21:53:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25700 IOs, 10880756736 bytes 2026-02-20T21:53:07.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25800 IOs, 10978991104 bytes 2026-02-20T21:53:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25900 IOs, 11033489408 bytes 2026-02-20T21:53:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26000 IOs, 11086217216 bytes 2026-02-20T21:53:07.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26100 IOs, 11171930112 bytes 2026-02-20T21:53:07.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26200 IOs, 11205454848 bytes 2026-02-20T21:53:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26300 IOs, 11206049792 bytes 2026-02-20T21:53:07.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26400 IOs, 11236168704 bytes 2026-02-20T21:53:07.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26500 IOs, 11248030720 bytes 2026-02-20T21:53:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26600 IOs, 11279751168 bytes 2026-02-20T21:53:07.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26700 IOs, 11341942784 bytes 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_32 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 snapshot 128M 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_32 128M 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_32 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_32 snapshot 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_32 snapshot 2026-02-20T21:53:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:53:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_32 2026-02-20T21:53:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_32': 2026-02-20T21:53:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:53:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:53:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:53:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 13fed4b9745 2026-02-20T21:53:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.13fed4b9745 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:07 2026 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:07 2026 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:07 2026 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 6f7f0c8e-e585-45a6-922c-3d3abe3d38f8 2026-02-20T21:53:08.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:53:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:53:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_32 2026-02-20T21:53:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T21:53:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:53:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=7 2026-02-20T21:53:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:53:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:53:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 20232192 bytes 2026-02-20T21:53:08.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 51659776 bytes 2026-02-20T21:53:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 77768704 bytes 2026-02-20T21:53:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 101257216 bytes 2026-02-20T21:53:08.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 134246400 bytes 2026-02-20T21:53:08.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 215943168 bytes 2026-02-20T21:53:08.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 216584192 bytes 2026-02-20T21:53:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 238592000 bytes 2026-02-20T21:53:09.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 333578240 bytes 2026-02-20T21:53:09.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 462780416 bytes 2026-02-20T21:53:09.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 467636224 bytes 2026-02-20T21:53:09.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 561364992 bytes 2026-02-20T21:53:09.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 624891904 bytes 2026-02-20T21:53:09.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 644588544 bytes 2026-02-20T21:53:09.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 654227456 bytes 2026-02-20T21:53:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 666151936 bytes 2026-02-20T21:53:09.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 775609344 bytes 2026-02-20T21:53:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 812849152 bytes 2026-02-20T21:53:09.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 880078848 bytes 2026-02-20T21:53:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 948046848 bytes 2026-02-20T21:53:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1002118144 bytes 2026-02-20T21:53:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1046131712 bytes 2026-02-20T21:53:09.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1098279936 bytes 2026-02-20T21:53:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1117326336 bytes 2026-02-20T21:53:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1164681216 bytes 2026-02-20T21:53:09.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1185136640 bytes 2026-02-20T21:53:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1227530240 bytes 2026-02-20T21:53:10.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1253604352 bytes 2026-02-20T21:53:10.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1281619968 bytes 2026-02-20T21:53:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1308867584 bytes 2026-02-20T21:53:10.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1336398848 bytes 2026-02-20T21:53:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1384408064 bytes 2026-02-20T21:53:10.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1425655808 bytes 2026-02-20T21:53:10.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1493800960 bytes 2026-02-20T21:53:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1523474432 bytes 2026-02-20T21:53:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1586856960 bytes 2026-02-20T21:53:10.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1638484992 bytes 2026-02-20T21:53:10.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1713010688 bytes 2026-02-20T21:53:10.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1767430144 bytes 2026-02-20T21:53:10.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1789264896 bytes 2026-02-20T21:53:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1838002176 bytes 2026-02-20T21:53:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1872547840 bytes 2026-02-20T21:53:10.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1981960192 bytes 2026-02-20T21:53:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2064780288 bytes 2026-02-20T21:53:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2186882048 bytes 2026-02-20T21:53:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2224729088 bytes 2026-02-20T21:53:11.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2245512192 bytes 2026-02-20T21:53:11.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2246131712 bytes 2026-02-20T21:53:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2297224192 bytes 2026-02-20T21:53:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2359053312 bytes 2026-02-20T21:53:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2377824256 bytes 2026-02-20T21:53:11.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2400144384 bytes 2026-02-20T21:53:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2446625792 bytes 2026-02-20T21:53:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2585629696 bytes 2026-02-20T21:53:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2623677440 bytes 2026-02-20T21:53:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2702333952 bytes 2026-02-20T21:53:11.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2724509696 bytes 2026-02-20T21:53:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2800847872 bytes 2026-02-20T21:53:11.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2870020096 bytes 2026-02-20T21:53:11.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2911822848 bytes 2026-02-20T21:53:11.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2966995968 bytes 2026-02-20T21:53:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 3012846592 bytes 2026-02-20T21:53:11.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 3027262464 bytes 2026-02-20T21:53:11.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3072743424 bytes 2026-02-20T21:53:11.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3084924928 bytes 2026-02-20T21:53:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3118315520 bytes 2026-02-20T21:53:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3181897728 bytes 2026-02-20T21:53:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3235087360 bytes 2026-02-20T21:53:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3267748864 bytes 2026-02-20T21:53:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3284060160 bytes 2026-02-20T21:53:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3306506240 bytes 2026-02-20T21:53:12.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3345406976 bytes 2026-02-20T21:53:12.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3372474368 bytes 2026-02-20T21:53:12.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3400448000 bytes 2026-02-20T21:53:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3427369984 bytes 2026-02-20T21:53:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3427989504 bytes 2026-02-20T21:53:12.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3462420480 bytes 2026-02-20T21:53:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3534649344 bytes 2026-02-20T21:53:12.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3559480320 bytes 2026-02-20T21:53:12.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3596180480 bytes 2026-02-20T21:53:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3681574912 bytes 2026-02-20T21:53:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3740499968 bytes 2026-02-20T21:53:12.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3782822912 bytes 2026-02-20T21:53:12.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3815941120 bytes 2026-02-20T21:53:12.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3831841792 bytes 2026-02-20T21:53:12.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3855106048 bytes 2026-02-20T21:53:12.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3919113216 bytes 2026-02-20T21:53:13.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4028297216 bytes 2026-02-20T21:53:13.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4036251648 bytes 2026-02-20T21:53:13.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4166155264 bytes 2026-02-20T21:53:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4197773312 bytes 2026-02-20T21:53:13.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4285422592 bytes 2026-02-20T21:53:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4321154048 bytes 2026-02-20T21:53:13.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4421629952 bytes 2026-02-20T21:53:13.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4484476928 bytes 2026-02-20T21:53:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4534424576 bytes 2026-02-20T21:53:13.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4565513216 bytes 2026-02-20T21:53:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4626212864 bytes 2026-02-20T21:53:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4649074688 bytes 2026-02-20T21:53:13.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4691937280 bytes 2026-02-20T21:53:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4773665792 bytes 2026-02-20T21:53:13.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4838942720 bytes 2026-02-20T21:53:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4862404608 bytes 2026-02-20T21:53:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4934334464 bytes 2026-02-20T21:53:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4957139968 bytes 2026-02-20T21:53:13.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4973673472 bytes 2026-02-20T21:53:14.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4997858304 bytes 2026-02-20T21:53:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 5030669312 bytes 2026-02-20T21:53:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 5075918848 bytes 2026-02-20T21:53:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5139380224 bytes 2026-02-20T21:53:14.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5140275200 bytes 2026-02-20T21:53:14.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5140935680 bytes 2026-02-20T21:53:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5181121536 bytes 2026-02-20T21:53:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5260093440 bytes 2026-02-20T21:53:14.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5325140992 bytes 2026-02-20T21:53:14.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5360963584 bytes 2026-02-20T21:53:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5409227776 bytes 2026-02-20T21:53:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5497798656 bytes 2026-02-20T21:53:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5533930496 bytes 2026-02-20T21:53:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5562328064 bytes 2026-02-20T21:53:14.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5615788032 bytes 2026-02-20T21:53:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5648769024 bytes 2026-02-20T21:53:14.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5673628672 bytes 2026-02-20T21:53:14.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5723929600 bytes 2026-02-20T21:53:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5786172416 bytes 2026-02-20T21:53:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5818940416 bytes 2026-02-20T21:53:15.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5851303936 bytes 2026-02-20T21:53:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5871920128 bytes 2026-02-20T21:53:15.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5920514048 bytes 2026-02-20T21:53:15.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5995239424 bytes 2026-02-20T21:53:15.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 6072571904 bytes 2026-02-20T21:53:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 6125338624 bytes 2026-02-20T21:53:15.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 6190483456 bytes 2026-02-20T21:53:15.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 6263344128 bytes 2026-02-20T21:53:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 6331082752 bytes 2026-02-20T21:53:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 6348525568 bytes 2026-02-20T21:53:15.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 6416128000 bytes 2026-02-20T21:53:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:53:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:53:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:53:15.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_33 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 snapshot 128M 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:53:15.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_33 128M 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:53:15.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:15.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_33 2026-02-20T21:53:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:53:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:53:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_33 snapshot 2026-02-20T21:53:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T21:53:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_33 snapshot 2026-02-20T21:53:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:53:16.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_33 2026-02-20T21:53:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_33': 2026-02-20T21:53:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:53:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:53:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:53:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 140a3f8ad78b 2026-02-20T21:53:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.140a3f8ad78b 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:15 2026 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:15 2026 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:15 2026 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:53:16.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 9a59b858-296a-4d6b-9785-36f62034ac7a 2026-02-20T21:53:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:53:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:53:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_33 2026-02-20T21:53:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T21:53:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:53:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:53:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:53:16.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:53:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 8289280 bytes 2026-02-20T21:53:16.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 8896512 bytes 2026-02-20T21:53:16.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 90180608 bytes 2026-02-20T21:53:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 136254464 bytes 2026-02-20T21:53:16.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 199853056 bytes 2026-02-20T21:53:16.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 200432640 bytes 2026-02-20T21:53:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 209529856 bytes 2026-02-20T21:53:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 253563904 bytes 2026-02-20T21:53:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 275392512 bytes 2026-02-20T21:53:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 399632384 bytes 2026-02-20T21:53:17.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 446954496 bytes 2026-02-20T21:53:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 485799936 bytes 2026-02-20T21:53:17.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 538139648 bytes 2026-02-20T21:53:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 551026688 bytes 2026-02-20T21:53:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 595435520 bytes 2026-02-20T21:53:17.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 617044992 bytes 2026-02-20T21:53:17.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 620505088 bytes 2026-02-20T21:53:17.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 649236480 bytes 2026-02-20T21:53:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 700357632 bytes 2026-02-20T21:53:17.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 720049152 bytes 2026-02-20T21:53:17.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 775939072 bytes 2026-02-20T21:53:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 827414528 bytes 2026-02-20T21:53:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 876873728 bytes 2026-02-20T21:53:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 965564416 bytes 2026-02-20T21:53:17.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 981750784 bytes 2026-02-20T21:53:17.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 997334016 bytes 2026-02-20T21:53:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1001613312 bytes 2026-02-20T21:53:17.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1044334592 bytes 2026-02-20T21:53:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1063441408 bytes 2026-02-20T21:53:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1090765824 bytes 2026-02-20T21:53:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1094607872 bytes 2026-02-20T21:53:18.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1182711808 bytes 2026-02-20T21:53:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1216741376 bytes 2026-02-20T21:53:18.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1230768128 bytes 2026-02-20T21:53:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1273450496 bytes 2026-02-20T21:53:18.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1288566784 bytes 2026-02-20T21:53:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1360832512 bytes 2026-02-20T21:53:18.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1414234112 bytes 2026-02-20T21:53:18.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1544082432 bytes 2026-02-20T21:53:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1605826560 bytes 2026-02-20T21:53:18.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1672412160 bytes 2026-02-20T21:53:18.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1731250176 bytes 2026-02-20T21:53:18.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1776989184 bytes 2026-02-20T21:53:18.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1830259712 bytes 2026-02-20T21:53:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1858221056 bytes 2026-02-20T21:53:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1882424320 bytes 2026-02-20T21:53:18.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1940706304 bytes 2026-02-20T21:53:19.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1944394752 bytes 2026-02-20T21:53:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1972969472 bytes 2026-02-20T21:53:19.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2022697984 bytes 2026-02-20T21:53:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2063897600 bytes 2026-02-20T21:53:19.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2133701632 bytes 2026-02-20T21:53:19.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2159758336 bytes 2026-02-20T21:53:19.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2186685440 bytes 2026-02-20T21:53:19.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2235852800 bytes 2026-02-20T21:53:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2252729344 bytes 2026-02-20T21:53:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2300705792 bytes 2026-02-20T21:53:19.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2340601856 bytes 2026-02-20T21:53:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2371129344 bytes 2026-02-20T21:53:19.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2404745216 bytes 2026-02-20T21:53:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2434898944 bytes 2026-02-20T21:53:19.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2454931456 bytes 2026-02-20T21:53:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2479979520 bytes 2026-02-20T21:53:19.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2494230528 bytes 2026-02-20T21:53:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2555227136 bytes 2026-02-20T21:53:19.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2555862016 bytes 2026-02-20T21:53:19.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2580280320 bytes 2026-02-20T21:53:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2635717632 bytes 2026-02-20T21:53:19.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2686657536 bytes 2026-02-20T21:53:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2698812416 bytes 2026-02-20T21:53:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2782771200 bytes 2026-02-20T21:53:20.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2795159552 bytes 2026-02-20T21:53:20.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2811804672 bytes 2026-02-20T21:53:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2856397824 bytes 2026-02-20T21:53:20.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2898756608 bytes 2026-02-20T21:53:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3019269120 bytes 2026-02-20T21:53:20.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3090425856 bytes 2026-02-20T21:53:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3168257024 bytes 2026-02-20T21:53:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3236256768 bytes 2026-02-20T21:53:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3248703488 bytes 2026-02-20T21:53:20.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3294661632 bytes 2026-02-20T21:53:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3382579200 bytes 2026-02-20T21:53:20.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3402637312 bytes 2026-02-20T21:53:20.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3452194816 bytes 2026-02-20T21:53:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3514189824 bytes 2026-02-20T21:53:20.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3576721408 bytes 2026-02-20T21:53:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3628833792 bytes 2026-02-20T21:53:20.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3661107200 bytes 2026-02-20T21:53:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3704574976 bytes 2026-02-20T21:53:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3750694912 bytes 2026-02-20T21:53:21.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3753535488 bytes 2026-02-20T21:53:21.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3790409728 bytes 2026-02-20T21:53:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3898447872 bytes 2026-02-20T21:53:21.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3950289920 bytes 2026-02-20T21:53:21.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4019222528 bytes 2026-02-20T21:53:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4060605440 bytes 2026-02-20T21:53:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4113668096 bytes 2026-02-20T21:53:21.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4130924544 bytes 2026-02-20T21:53:21.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4171642880 bytes 2026-02-20T21:53:21.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4193007616 bytes 2026-02-20T21:53:21.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4243625984 bytes 2026-02-20T21:53:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4307951616 bytes 2026-02-20T21:53:21.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4308582400 bytes 2026-02-20T21:53:21.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4334521344 bytes 2026-02-20T21:53:21.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4424571904 bytes 2026-02-20T21:53:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4435198976 bytes 2026-02-20T21:53:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4451500032 bytes 2026-02-20T21:53:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4534727680 bytes 2026-02-20T21:53:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4564933632 bytes 2026-02-20T21:53:22.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4577547264 bytes 2026-02-20T21:53:22.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4592492544 bytes 2026-02-20T21:53:22.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4625021952 bytes 2026-02-20T21:53:22.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4699963392 bytes 2026-02-20T21:53:22.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4799471616 bytes 2026-02-20T21:53:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4832575488 bytes 2026-02-20T21:53:22.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4845814784 bytes 2026-02-20T21:53:22.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4867319808 bytes 2026-02-20T21:53:22.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4945994752 bytes 2026-02-20T21:53:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4971215872 bytes 2026-02-20T21:53:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5069284352 bytes 2026-02-20T21:53:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:53:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:53:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:53:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:53:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_34 2026-02-20T21:53:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 snapshot 128M 2026-02-20T21:53:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_34 128M 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T21:53:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:53:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:53:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:53:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_34 2026-02-20T21:53:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:53:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:22.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:53:22.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:22.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_34 snapshot 2026-02-20T21:53:22.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:22.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:22.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T21:53:22.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:22.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_34 snapshot 2026-02-20T21:53:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:53:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_34 2026-02-20T21:53:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_34': 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 14166e743fdd 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.14166e743fdd 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:53:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:22 2026 2026-02-20T21:53:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:22 2026 2026-02-20T21:53:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:22 2026 2026-02-20T21:53:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:53:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:53:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 843abf52-674b-43d7-b959-4287ddcbd20a 2026-02-20T21:53:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:53:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:53:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_34 2026-02-20T21:53:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T21:53:23.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:53:23.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T21:53:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:53:23.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:53:23.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 15812608 bytes 2026-02-20T21:53:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 26860544 bytes 2026-02-20T21:53:23.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 44887040 bytes 2026-02-20T21:53:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 45932544 bytes 2026-02-20T21:53:23.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 115036160 bytes 2026-02-20T21:53:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 126928896 bytes 2026-02-20T21:53:23.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 151616512 bytes 2026-02-20T21:53:24.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 246494208 bytes 2026-02-20T21:53:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 294421504 bytes 2026-02-20T21:53:24.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 295040000 bytes 2026-02-20T21:53:24.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 366840832 bytes 2026-02-20T21:53:24.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 418103296 bytes 2026-02-20T21:53:24.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 483908608 bytes 2026-02-20T21:53:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 522957824 bytes 2026-02-20T21:53:24.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 576901120 bytes 2026-02-20T21:53:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 623408128 bytes 2026-02-20T21:53:24.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 663958528 bytes 2026-02-20T21:53:24.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 674866176 bytes 2026-02-20T21:53:24.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 706093056 bytes 2026-02-20T21:53:24.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 759620608 bytes 2026-02-20T21:53:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 820999168 bytes 2026-02-20T21:53:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 855107584 bytes 2026-02-20T21:53:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 893960192 bytes 2026-02-20T21:53:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 977550336 bytes 2026-02-20T21:53:24.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1018318848 bytes 2026-02-20T21:53:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1018925056 bytes 2026-02-20T21:53:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1054947328 bytes 2026-02-20T21:53:24.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1103187968 bytes 2026-02-20T21:53:25.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1149658112 bytes 2026-02-20T21:53:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1205351424 bytes 2026-02-20T21:53:25.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1229990912 bytes 2026-02-20T21:53:25.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1249186816 bytes 2026-02-20T21:53:25.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1289357312 bytes 2026-02-20T21:53:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1301266432 bytes 2026-02-20T21:53:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1330579456 bytes 2026-02-20T21:53:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1384373248 bytes 2026-02-20T21:53:25.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1412894720 bytes 2026-02-20T21:53:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1416653824 bytes 2026-02-20T21:53:25.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1439671296 bytes 2026-02-20T21:53:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1478321152 bytes 2026-02-20T21:53:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1531836416 bytes 2026-02-20T21:53:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1559860224 bytes 2026-02-20T21:53:25.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1633113088 bytes 2026-02-20T21:53:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1669017600 bytes 2026-02-20T21:53:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1677463552 bytes 2026-02-20T21:53:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1724932096 bytes 2026-02-20T21:53:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1751392256 bytes 2026-02-20T21:53:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1797285888 bytes 2026-02-20T21:53:25.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1821457408 bytes 2026-02-20T21:53:25.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1867231232 bytes 2026-02-20T21:53:26.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 1882639360 bytes 2026-02-20T21:53:26.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 1919348736 bytes 2026-02-20T21:53:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 1958335488 bytes 2026-02-20T21:53:26.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 1997279232 bytes 2026-02-20T21:53:26.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2070187008 bytes 2026-02-20T21:53:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2093716480 bytes 2026-02-20T21:53:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2139524096 bytes 2026-02-20T21:53:26.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2191593472 bytes 2026-02-20T21:53:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2244519936 bytes 2026-02-20T21:53:26.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2262139904 bytes 2026-02-20T21:53:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2276243456 bytes 2026-02-20T21:53:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2291875840 bytes 2026-02-20T21:53:26.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2371991552 bytes 2026-02-20T21:53:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2372630528 bytes 2026-02-20T21:53:26.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2397050880 bytes 2026-02-20T21:53:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2442255360 bytes 2026-02-20T21:53:26.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2463057920 bytes 2026-02-20T21:53:26.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2529206272 bytes 2026-02-20T21:53:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2605147136 bytes 2026-02-20T21:53:26.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2701312000 bytes 2026-02-20T21:53:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2734721024 bytes 2026-02-20T21:53:27.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2736025600 bytes 2026-02-20T21:53:27.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2789424128 bytes 2026-02-20T21:53:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2790031360 bytes 2026-02-20T21:53:27.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2816535552 bytes 2026-02-20T21:53:27.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 2830291968 bytes 2026-02-20T21:53:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 2872671232 bytes 2026-02-20T21:53:27.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 2984238080 bytes 2026-02-20T21:53:27.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3000244224 bytes 2026-02-20T21:53:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3035915264 bytes 2026-02-20T21:53:27.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3128378368 bytes 2026-02-20T21:53:27.525 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9826970694280215) > weight (0.25) 2026-02-20T21:53:27.525 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5290392175133384) > weight (0.25) 2026-02-20T21:53:27.525 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.7622931838346511) > weight (0.25) 2026-02-20T21:53:27.525 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.47472627251362076) > weight (0.25) 2026-02-20T21:53:27.525 INFO:tasks.rbd_mirror_thrash:waiting for 104.8566788935277 secs before thrashing 2026-02-20T21:53:27.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3183926272 bytes 2026-02-20T21:53:27.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3219243008 bytes 2026-02-20T21:53:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3258547200 bytes 2026-02-20T21:53:27.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3327569920 bytes 2026-02-20T21:53:27.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3386551296 bytes 2026-02-20T21:53:27.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3403574272 bytes 2026-02-20T21:53:27.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3413102592 bytes 2026-02-20T21:53:27.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3431968768 bytes 2026-02-20T21:53:27.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3451347968 bytes 2026-02-20T21:53:27.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3458386944 bytes 2026-02-20T21:53:27.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3489107968 bytes 2026-02-20T21:53:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3536230400 bytes 2026-02-20T21:53:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3598923776 bytes 2026-02-20T21:53:28.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3629108224 bytes 2026-02-20T21:53:28.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3667329024 bytes 2026-02-20T21:53:28.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3757025280 bytes 2026-02-20T21:53:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3786059776 bytes 2026-02-20T21:53:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3801915392 bytes 2026-02-20T21:53:28.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3869249536 bytes 2026-02-20T21:53:28.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4020041728 bytes 2026-02-20T21:53:28.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4072019968 bytes 2026-02-20T21:53:28.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4089086976 bytes 2026-02-20T21:53:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4124913664 bytes 2026-02-20T21:53:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4171777024 bytes 2026-02-20T21:53:28.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4269203456 bytes 2026-02-20T21:53:28.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4299968512 bytes 2026-02-20T21:53:28.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4305561600 bytes 2026-02-20T21:53:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4341665792 bytes 2026-02-20T21:53:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4342258688 bytes 2026-02-20T21:53:28.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4382799872 bytes 2026-02-20T21:53:28.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4438921216 bytes 2026-02-20T21:53:28.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4475318272 bytes 2026-02-20T21:53:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4501939200 bytes 2026-02-20T21:53:29.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4570407936 bytes 2026-02-20T21:53:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4623087616 bytes 2026-02-20T21:53:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4647078912 bytes 2026-02-20T21:53:29.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4693341184 bytes 2026-02-20T21:53:29.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4693978112 bytes 2026-02-20T21:53:29.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4710908928 bytes 2026-02-20T21:53:29.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4754280448 bytes 2026-02-20T21:53:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4784171008 bytes 2026-02-20T21:53:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4836378624 bytes 2026-02-20T21:53:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4897975296 bytes 2026-02-20T21:53:29.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4988602368 bytes 2026-02-20T21:53:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5053020160 bytes 2026-02-20T21:53:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5080339456 bytes 2026-02-20T21:53:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5120072704 bytes 2026-02-20T21:53:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5144452096 bytes 2026-02-20T21:53:29.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5191540736 bytes 2026-02-20T21:53:29.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5254506496 bytes 2026-02-20T21:53:29.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5317265408 bytes 2026-02-20T21:53:29.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5428322304 bytes 2026-02-20T21:53:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5468231680 bytes 2026-02-20T21:53:30.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5525091328 bytes 2026-02-20T21:53:30.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5525702656 bytes 2026-02-20T21:53:30.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5574810624 bytes 2026-02-20T21:53:30.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5591265280 bytes 2026-02-20T21:53:30.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5633991680 bytes 2026-02-20T21:53:30.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5644625920 bytes 2026-02-20T21:53:30.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5723098112 bytes 2026-02-20T21:53:30.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5735791616 bytes 2026-02-20T21:53:30.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5777941504 bytes 2026-02-20T21:53:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5786688512 bytes 2026-02-20T21:53:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5839227904 bytes 2026-02-20T21:53:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5861160960 bytes 2026-02-20T21:53:30.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5881911296 bytes 2026-02-20T21:53:30.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5894171648 bytes 2026-02-20T21:53:30.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5942759424 bytes 2026-02-20T21:53:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5999438848 bytes 2026-02-20T21:53:30.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6048517120 bytes 2026-02-20T21:53:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6112753664 bytes 2026-02-20T21:53:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6157126656 bytes 2026-02-20T21:53:30.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6191741952 bytes 2026-02-20T21:53:30.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6197535744 bytes 2026-02-20T21:53:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6210826240 bytes 2026-02-20T21:53:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6237391872 bytes 2026-02-20T21:53:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6264081408 bytes 2026-02-20T21:53:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6275912704 bytes 2026-02-20T21:53:31.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6318538752 bytes 2026-02-20T21:53:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6330788864 bytes 2026-02-20T21:53:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6358602752 bytes 2026-02-20T21:53:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6385927168 bytes 2026-02-20T21:53:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6478509056 bytes 2026-02-20T21:53:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6491315200 bytes 2026-02-20T21:53:31.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6542728192 bytes 2026-02-20T21:53:31.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6565185536 bytes 2026-02-20T21:53:31.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6573556736 bytes 2026-02-20T21:53:31.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6580113408 bytes 2026-02-20T21:53:31.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6633169920 bytes 2026-02-20T21:53:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6639887360 bytes 2026-02-20T21:53:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6713288704 bytes 2026-02-20T21:53:31.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 6747220992 bytes 2026-02-20T21:53:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 6831782912 bytes 2026-02-20T21:53:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 6868317184 bytes 2026-02-20T21:53:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 6948950016 bytes 2026-02-20T21:53:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 6973755392 bytes 2026-02-20T21:53:32.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7000580096 bytes 2026-02-20T21:53:32.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7083829248 bytes 2026-02-20T21:53:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7132222464 bytes 2026-02-20T21:53:32.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7158590464 bytes 2026-02-20T21:53:32.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7207076864 bytes 2026-02-20T21:53:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7239165952 bytes 2026-02-20T21:53:32.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7281613824 bytes 2026-02-20T21:53:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7293913088 bytes 2026-02-20T21:53:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:53:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_35 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 snapshot 128M 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:53:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_35 128M 2026-02-20T21:53:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T21:53:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:53:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:53:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:53:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_35 2026-02-20T21:53:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:53:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:53:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_35 snapshot 2026-02-20T21:53:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T21:53:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_35 snapshot 2026-02-20T21:53:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:53:33.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_35 2026-02-20T21:53:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_35': 2026-02-20T21:53:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 14221bc7c0ef 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.14221bc7c0ef 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:32 2026 2026-02-20T21:53:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:32 2026 2026-02-20T21:53:33.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:32 2026 2026-02-20T21:53:33.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:53:33.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:53:33.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 15ffbe72-993b-43ba-9ff0-25518ecb6ae4 2026-02-20T21:53:33.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:53:33.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:53:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_35 2026-02-20T21:53:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T21:53:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:53:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=12 2026-02-20T21:53:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:53:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:53:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 71972864 bytes 2026-02-20T21:53:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 139902976 bytes 2026-02-20T21:53:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 234989568 bytes 2026-02-20T21:53:34.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 290672640 bytes 2026-02-20T21:53:34.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 346814464 bytes 2026-02-20T21:53:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 397595648 bytes 2026-02-20T21:53:34.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 458787840 bytes 2026-02-20T21:53:34.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 484145152 bytes 2026-02-20T21:53:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 508883968 bytes 2026-02-20T21:53:34.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 538567680 bytes 2026-02-20T21:53:34.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 565794816 bytes 2026-02-20T21:53:34.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 566412288 bytes 2026-02-20T21:53:34.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 621329408 bytes 2026-02-20T21:53:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 638642176 bytes 2026-02-20T21:53:34.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 677385216 bytes 2026-02-20T21:53:34.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 764844032 bytes 2026-02-20T21:53:34.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 826070016 bytes 2026-02-20T21:53:34.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 950801408 bytes 2026-02-20T21:53:34.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 971805696 bytes 2026-02-20T21:53:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 997528576 bytes 2026-02-20T21:53:35.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1052044288 bytes 2026-02-20T21:53:35.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1089638400 bytes 2026-02-20T21:53:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1135797248 bytes 2026-02-20T21:53:35.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1152802816 bytes 2026-02-20T21:53:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1167116288 bytes 2026-02-20T21:53:35.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1255012352 bytes 2026-02-20T21:53:35.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1319585792 bytes 2026-02-20T21:53:35.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1341629440 bytes 2026-02-20T21:53:35.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1391296512 bytes 2026-02-20T21:53:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1418573824 bytes 2026-02-20T21:53:35.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1443169280 bytes 2026-02-20T21:53:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1472014336 bytes 2026-02-20T21:53:35.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1572661248 bytes 2026-02-20T21:53:35.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1605295104 bytes 2026-02-20T21:53:35.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1651425280 bytes 2026-02-20T21:53:35.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1703261184 bytes 2026-02-20T21:53:35.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1750176768 bytes 2026-02-20T21:53:35.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1820417024 bytes 2026-02-20T21:53:35.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1847945216 bytes 2026-02-20T21:53:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1931533312 bytes 2026-02-20T21:53:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1990179840 bytes 2026-02-20T21:53:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2033148928 bytes 2026-02-20T21:53:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2058741760 bytes 2026-02-20T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2097932288 bytes 2026-02-20T21:53:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2116440064 bytes 2026-02-20T21:53:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2149645312 bytes 2026-02-20T21:53:36.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2150255616 bytes 2026-02-20T21:53:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2265268224 bytes 2026-02-20T21:53:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2290350080 bytes 2026-02-20T21:53:36.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2329282560 bytes 2026-02-20T21:53:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2357609472 bytes 2026-02-20T21:53:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2376657920 bytes 2026-02-20T21:53:36.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2399302656 bytes 2026-02-20T21:53:36.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2454852608 bytes 2026-02-20T21:53:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2486453248 bytes 2026-02-20T21:53:36.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2559989760 bytes 2026-02-20T21:53:36.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2579539968 bytes 2026-02-20T21:53:36.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2610833408 bytes 2026-02-20T21:53:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2633819136 bytes 2026-02-20T21:53:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2647716864 bytes 2026-02-20T21:53:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2664860672 bytes 2026-02-20T21:53:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2722339840 bytes 2026-02-20T21:53:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2761351168 bytes 2026-02-20T21:53:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2821138432 bytes 2026-02-20T21:53:37.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2862842880 bytes 2026-02-20T21:53:37.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2924559360 bytes 2026-02-20T21:53:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2925158400 bytes 2026-02-20T21:53:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2944346112 bytes 2026-02-20T21:53:37.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2999363584 bytes 2026-02-20T21:53:37.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3061079040 bytes 2026-02-20T21:53:37.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3065517056 bytes 2026-02-20T21:53:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3097826304 bytes 2026-02-20T21:53:37.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3098555392 bytes 2026-02-20T21:53:37.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3178110976 bytes 2026-02-20T21:53:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3194029056 bytes 2026-02-20T21:53:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3245325312 bytes 2026-02-20T21:53:37.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3277284352 bytes 2026-02-20T21:53:37.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3350334464 bytes 2026-02-20T21:53:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3410344960 bytes 2026-02-20T21:53:37.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3435452416 bytes 2026-02-20T21:53:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3465353216 bytes 2026-02-20T21:53:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3506138112 bytes 2026-02-20T21:53:37.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3514455040 bytes 2026-02-20T21:53:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3526552576 bytes 2026-02-20T21:53:38.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3541154816 bytes 2026-02-20T21:53:38.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3581634560 bytes 2026-02-20T21:53:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3646346240 bytes 2026-02-20T21:53:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3698910208 bytes 2026-02-20T21:53:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3739502592 bytes 2026-02-20T21:53:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3776482304 bytes 2026-02-20T21:53:38.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3800939520 bytes 2026-02-20T21:53:38.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3812672512 bytes 2026-02-20T21:53:38.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3906013184 bytes 2026-02-20T21:53:38.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3946963968 bytes 2026-02-20T21:53:38.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3969561600 bytes 2026-02-20T21:53:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4009661440 bytes 2026-02-20T21:53:38.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4097492992 bytes 2026-02-20T21:53:38.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4121104384 bytes 2026-02-20T21:53:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4121690112 bytes 2026-02-20T21:53:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4122305536 bytes 2026-02-20T21:53:38.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4139216896 bytes 2026-02-20T21:53:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4179878912 bytes 2026-02-20T21:53:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4220628992 bytes 2026-02-20T21:53:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4281189376 bytes 2026-02-20T21:53:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4296291328 bytes 2026-02-20T21:53:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4375240704 bytes 2026-02-20T21:53:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4400101376 bytes 2026-02-20T21:53:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4435203072 bytes 2026-02-20T21:53:39.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4498971648 bytes 2026-02-20T21:53:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4573376512 bytes 2026-02-20T21:53:39.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4623581184 bytes 2026-02-20T21:53:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4690421760 bytes 2026-02-20T21:53:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4715909120 bytes 2026-02-20T21:53:39.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4746782720 bytes 2026-02-20T21:53:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4818908160 bytes 2026-02-20T21:53:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4868014080 bytes 2026-02-20T21:53:39.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4899066880 bytes 2026-02-20T21:53:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4930311168 bytes 2026-02-20T21:53:39.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4975198208 bytes 2026-02-20T21:53:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5064753152 bytes 2026-02-20T21:53:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5119155200 bytes 2026-02-20T21:53:39.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5155307520 bytes 2026-02-20T21:53:39.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5233344512 bytes 2026-02-20T21:53:39.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5270228992 bytes 2026-02-20T21:53:40.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5367546880 bytes 2026-02-20T21:53:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5403888640 bytes 2026-02-20T21:53:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5427058688 bytes 2026-02-20T21:53:40.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5449498624 bytes 2026-02-20T21:53:40.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5454208000 bytes 2026-02-20T21:53:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5491840000 bytes 2026-02-20T21:53:40.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5544096768 bytes 2026-02-20T21:53:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5588020224 bytes 2026-02-20T21:53:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5648399360 bytes 2026-02-20T21:53:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5672663040 bytes 2026-02-20T21:53:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5713793024 bytes 2026-02-20T21:53:40.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5825767424 bytes 2026-02-20T21:53:40.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5872954368 bytes 2026-02-20T21:53:40.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5891933184 bytes 2026-02-20T21:53:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5955691520 bytes 2026-02-20T21:53:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5974230016 bytes 2026-02-20T21:53:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6030687232 bytes 2026-02-20T21:53:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6055947264 bytes 2026-02-20T21:53:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6101204992 bytes 2026-02-20T21:53:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6101813248 bytes 2026-02-20T21:53:40.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6169059328 bytes 2026-02-20T21:53:41.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6234400768 bytes 2026-02-20T21:53:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6261749760 bytes 2026-02-20T21:53:41.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6319966208 bytes 2026-02-20T21:53:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6389772288 bytes 2026-02-20T21:53:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6437038080 bytes 2026-02-20T21:53:41.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6499426304 bytes 2026-02-20T21:53:41.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6548477952 bytes 2026-02-20T21:53:41.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6567500800 bytes 2026-02-20T21:53:41.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6606316544 bytes 2026-02-20T21:53:41.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6692160512 bytes 2026-02-20T21:53:41.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6733702144 bytes 2026-02-20T21:53:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6771364864 bytes 2026-02-20T21:53:41.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6816900096 bytes 2026-02-20T21:53:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6939337728 bytes 2026-02-20T21:53:41.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6961547264 bytes 2026-02-20T21:53:41.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7027030016 bytes 2026-02-20T21:53:41.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7050385408 bytes 2026-02-20T21:53:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7108084736 bytes 2026-02-20T21:53:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7200146432 bytes 2026-02-20T21:53:42.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7274797056 bytes 2026-02-20T21:53:42.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7276402688 bytes 2026-02-20T21:53:42.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7295750144 bytes 2026-02-20T21:53:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7333938176 bytes 2026-02-20T21:53:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7354190848 bytes 2026-02-20T21:53:42.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7378206720 bytes 2026-02-20T21:53:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7417321472 bytes 2026-02-20T21:53:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7481207808 bytes 2026-02-20T21:53:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7518770176 bytes 2026-02-20T21:53:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7558396928 bytes 2026-02-20T21:53:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7585434624 bytes 2026-02-20T21:53:42.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7669596160 bytes 2026-02-20T21:53:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7690390528 bytes 2026-02-20T21:53:42.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7733094400 bytes 2026-02-20T21:53:42.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7751196672 bytes 2026-02-20T21:53:42.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7873677312 bytes 2026-02-20T21:53:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7917462528 bytes 2026-02-20T21:53:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7935407104 bytes 2026-02-20T21:53:42.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7981780992 bytes 2026-02-20T21:53:42.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 8028376064 bytes 2026-02-20T21:53:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 8144089088 bytes 2026-02-20T21:53:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 8207249408 bytes 2026-02-20T21:53:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8226008064 bytes 2026-02-20T21:53:43.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8265192448 bytes 2026-02-20T21:53:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8298691584 bytes 2026-02-20T21:53:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8333126656 bytes 2026-02-20T21:53:43.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8365602816 bytes 2026-02-20T21:53:43.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8377339904 bytes 2026-02-20T21:53:43.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8413874176 bytes 2026-02-20T21:53:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8444524544 bytes 2026-02-20T21:53:43.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8472240128 bytes 2026-02-20T21:53:43.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8539145216 bytes 2026-02-20T21:53:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8576417792 bytes 2026-02-20T21:53:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8626887680 bytes 2026-02-20T21:53:43.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8678924288 bytes 2026-02-20T21:53:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8707928064 bytes 2026-02-20T21:53:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8809239552 bytes 2026-02-20T21:53:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8858106880 bytes 2026-02-20T21:53:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8858715136 bytes 2026-02-20T21:53:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8888161280 bytes 2026-02-20T21:53:44.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8963362816 bytes 2026-02-20T21:53:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 9042666496 bytes 2026-02-20T21:53:44.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 9063387136 bytes 2026-02-20T21:53:44.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 9099696128 bytes 2026-02-20T21:53:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 9139398656 bytes 2026-02-20T21:53:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9225875456 bytes 2026-02-20T21:53:44.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9315756032 bytes 2026-02-20T21:53:44.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9381271552 bytes 2026-02-20T21:53:44.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9431795712 bytes 2026-02-20T21:53:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9475008512 bytes 2026-02-20T21:53:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9550295040 bytes 2026-02-20T21:53:44.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9572774912 bytes 2026-02-20T21:53:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9596568576 bytes 2026-02-20T21:53:44.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9709637632 bytes 2026-02-20T21:53:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9729361920 bytes 2026-02-20T21:53:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9782567936 bytes 2026-02-20T21:53:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9869893632 bytes 2026-02-20T21:53:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9912111104 bytes 2026-02-20T21:53:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9950301184 bytes 2026-02-20T21:53:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9970717696 bytes 2026-02-20T21:53:45.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 10011627520 bytes 2026-02-20T21:53:45.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 10085464064 bytes 2026-02-20T21:53:45.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 10096392192 bytes 2026-02-20T21:53:45.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 10119887872 bytes 2026-02-20T21:53:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 10173188096 bytes 2026-02-20T21:53:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 10258307072 bytes 2026-02-20T21:53:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 10268883968 bytes 2026-02-20T21:53:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 10299967488 bytes 2026-02-20T21:53:45.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 10365458432 bytes 2026-02-20T21:53:45.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 10412218368 bytes 2026-02-20T21:53:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 10422653952 bytes 2026-02-20T21:53:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 10469451776 bytes 2026-02-20T21:53:45.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 10511460352 bytes 2026-02-20T21:53:45.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 10554642432 bytes 2026-02-20T21:53:45.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 10630486016 bytes 2026-02-20T21:53:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 10673745920 bytes 2026-02-20T21:53:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_36 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 snapshot 128M 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_36 128M 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T21:53:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:53:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:53:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:53:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_36 2026-02-20T21:53:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:53:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:53:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_36 snapshot 2026-02-20T21:53:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T21:53:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_36 snapshot 2026-02-20T21:53:46.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:53:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_36 2026-02-20T21:53:46.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_36': 2026-02-20T21:53:46.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:53:46.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:53:46.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:53:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 142e83a93d5d 2026-02-20T21:53:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.142e83a93d5d 2026-02-20T21:53:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:53:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:53:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:53:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:53:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:45 2026 2026-02-20T21:53:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:45 2026 2026-02-20T21:53:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:45 2026 2026-02-20T21:53:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:53:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:53:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: cb656abb-b533-4a24-8070-aceb29417b9d 2026-02-20T21:53:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:53:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:53:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_36 2026-02-20T21:53:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T21:53:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:53:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:53:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:53:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:53:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 48605184 bytes 2026-02-20T21:53:46.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 59628544 bytes 2026-02-20T21:53:47.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 89176064 bytes 2026-02-20T21:53:47.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 138429440 bytes 2026-02-20T21:53:47.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 194516992 bytes 2026-02-20T21:53:47.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 214116352 bytes 2026-02-20T21:53:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 226750464 bytes 2026-02-20T21:53:47.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 276667392 bytes 2026-02-20T21:53:47.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 303217664 bytes 2026-02-20T21:53:47.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 364282880 bytes 2026-02-20T21:53:47.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 422062080 bytes 2026-02-20T21:53:47.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 465534976 bytes 2026-02-20T21:53:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 477729792 bytes 2026-02-20T21:53:47.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 514664448 bytes 2026-02-20T21:53:47.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 552869888 bytes 2026-02-20T21:53:47.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 654763008 bytes 2026-02-20T21:53:47.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 671290368 bytes 2026-02-20T21:53:47.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 678362112 bytes 2026-02-20T21:53:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 758632448 bytes 2026-02-20T21:53:47.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 815859712 bytes 2026-02-20T21:53:48.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 887963648 bytes 2026-02-20T21:53:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 925859840 bytes 2026-02-20T21:53:48.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 955389952 bytes 2026-02-20T21:53:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 980245504 bytes 2026-02-20T21:53:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1002011648 bytes 2026-02-20T21:53:48.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1033240576 bytes 2026-02-20T21:53:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1049090048 bytes 2026-02-20T21:53:48.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1063795712 bytes 2026-02-20T21:53:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1118856192 bytes 2026-02-20T21:53:48.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1147223040 bytes 2026-02-20T21:53:48.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1147822080 bytes 2026-02-20T21:53:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1299866624 bytes 2026-02-20T21:53:48.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1321078784 bytes 2026-02-20T21:53:48.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1350577152 bytes 2026-02-20T21:53:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1356407808 bytes 2026-02-20T21:53:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1369262080 bytes 2026-02-20T21:53:48.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1433961472 bytes 2026-02-20T21:53:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1477719040 bytes 2026-02-20T21:53:48.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1516125184 bytes 2026-02-20T21:53:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1578098688 bytes 2026-02-20T21:53:48.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1581822976 bytes 2026-02-20T21:53:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1582987264 bytes 2026-02-20T21:53:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1623228416 bytes 2026-02-20T21:53:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1712386048 bytes 2026-02-20T21:53:49.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1771731968 bytes 2026-02-20T21:53:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1821750272 bytes 2026-02-20T21:53:49.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1878797312 bytes 2026-02-20T21:53:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1985598464 bytes 2026-02-20T21:53:49.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2004492288 bytes 2026-02-20T21:53:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2041617408 bytes 2026-02-20T21:53:49.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2095149056 bytes 2026-02-20T21:53:49.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2187507712 bytes 2026-02-20T21:53:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2226784256 bytes 2026-02-20T21:53:49.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2280852480 bytes 2026-02-20T21:53:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2346968064 bytes 2026-02-20T21:53:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2373506048 bytes 2026-02-20T21:53:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2411154432 bytes 2026-02-20T21:53:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2552147968 bytes 2026-02-20T21:53:49.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2553123840 bytes 2026-02-20T21:53:49.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2590068736 bytes 2026-02-20T21:53:50.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2677517312 bytes 2026-02-20T21:53:50.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2699691008 bytes 2026-02-20T21:53:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2743584768 bytes 2026-02-20T21:53:50.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2749078528 bytes 2026-02-20T21:53:50.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2816754688 bytes 2026-02-20T21:53:50.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2884507648 bytes 2026-02-20T21:53:50.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2933170176 bytes 2026-02-20T21:53:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2969077760 bytes 2026-02-20T21:53:50.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3063651328 bytes 2026-02-20T21:53:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3084238848 bytes 2026-02-20T21:53:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3166386176 bytes 2026-02-20T21:53:50.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3212839936 bytes 2026-02-20T21:53:50.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3244677120 bytes 2026-02-20T21:53:50.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3260070912 bytes 2026-02-20T21:53:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3280509952 bytes 2026-02-20T21:53:50.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3312228352 bytes 2026-02-20T21:53:50.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3396926464 bytes 2026-02-20T21:53:50.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3435300864 bytes 2026-02-20T21:53:50.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3480424448 bytes 2026-02-20T21:53:51.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3540845568 bytes 2026-02-20T21:53:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3610018816 bytes 2026-02-20T21:53:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3659694080 bytes 2026-02-20T21:53:51.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3716046848 bytes 2026-02-20T21:53:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3742651392 bytes 2026-02-20T21:53:51.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3801109504 bytes 2026-02-20T21:53:51.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3821946880 bytes 2026-02-20T21:53:51.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3897637888 bytes 2026-02-20T21:53:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3933782016 bytes 2026-02-20T21:53:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4015944704 bytes 2026-02-20T21:53:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4016567296 bytes 2026-02-20T21:53:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4120199168 bytes 2026-02-20T21:53:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4171741184 bytes 2026-02-20T21:53:51.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4210048000 bytes 2026-02-20T21:53:51.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4235668480 bytes 2026-02-20T21:53:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4303532032 bytes 2026-02-20T21:53:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4320180224 bytes 2026-02-20T21:53:51.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4399602688 bytes 2026-02-20T21:53:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4448942080 bytes 2026-02-20T21:53:51.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4458550272 bytes 2026-02-20T21:53:52.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4459155456 bytes 2026-02-20T21:53:52.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4555885568 bytes 2026-02-20T21:53:52.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4688082944 bytes 2026-02-20T21:53:52.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4720315392 bytes 2026-02-20T21:53:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4765544448 bytes 2026-02-20T21:53:52.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4786699264 bytes 2026-02-20T21:53:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4810435584 bytes 2026-02-20T21:53:52.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4863124480 bytes 2026-02-20T21:53:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4888955904 bytes 2026-02-20T21:53:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4893603840 bytes 2026-02-20T21:53:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4984824832 bytes 2026-02-20T21:53:52.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5024588800 bytes 2026-02-20T21:53:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5090031616 bytes 2026-02-20T21:53:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:53:52.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_37 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 snapshot 128M 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_37 128M 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:53:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:53:52.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:53:52.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_37 2026-02-20T21:53:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:53:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:53:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:53:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_37 snapshot 2026-02-20T21:53:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T21:53:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:53:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_37 snapshot 2026-02-20T21:53:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:53:53.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_37 2026-02-20T21:53:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_37': 2026-02-20T21:53:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 143a82a2faa8 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.143a82a2faa8 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:52 2026 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:52 2026 2026-02-20T21:53:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:52 2026 2026-02-20T21:53:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:53:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:53:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 433c55c5-23ec-4be7-a8a5-59991ae7abd8 2026-02-20T21:53:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:53:53.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:53:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_37 2026-02-20T21:53:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:53:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:53:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T21:53:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:53:53.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=10 2026-02-20T21:53:53.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:53:53.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:53:53.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 8033280 bytes 2026-02-20T21:53:53.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 91583488 bytes 2026-02-20T21:53:53.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 112253952 bytes 2026-02-20T21:53:53.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 152450048 bytes 2026-02-20T21:53:54.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 163289088 bytes 2026-02-20T21:53:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 197317632 bytes 2026-02-20T21:53:54.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 218713088 bytes 2026-02-20T21:53:54.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 260573184 bytes 2026-02-20T21:53:54.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 280988672 bytes 2026-02-20T21:53:54.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 295883776 bytes 2026-02-20T21:53:54.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 316252160 bytes 2026-02-20T21:53:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 422018048 bytes 2026-02-20T21:53:54.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 442263552 bytes 2026-02-20T21:53:54.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 463846400 bytes 2026-02-20T21:53:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 503727104 bytes 2026-02-20T21:53:54.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 523407360 bytes 2026-02-20T21:53:54.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 614073344 bytes 2026-02-20T21:53:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 768903168 bytes 2026-02-20T21:53:54.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 807102464 bytes 2026-02-20T21:53:54.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 821020672 bytes 2026-02-20T21:53:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 824423424 bytes 2026-02-20T21:53:54.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 865219584 bytes 2026-02-20T21:53:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 897293312 bytes 2026-02-20T21:53:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 920854528 bytes 2026-02-20T21:53:54.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 935294976 bytes 2026-02-20T21:53:55.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 989086720 bytes 2026-02-20T21:53:55.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1060725760 bytes 2026-02-20T21:53:55.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1072820224 bytes 2026-02-20T21:53:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1132165120 bytes 2026-02-20T21:53:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1178398720 bytes 2026-02-20T21:53:55.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1182623744 bytes 2026-02-20T21:53:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1216687104 bytes 2026-02-20T21:53:55.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1275463680 bytes 2026-02-20T21:53:55.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1326203904 bytes 2026-02-20T21:53:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1345114112 bytes 2026-02-20T21:53:55.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1384974336 bytes 2026-02-20T21:53:55.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1407261696 bytes 2026-02-20T21:53:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1447924736 bytes 2026-02-20T21:53:55.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1502021632 bytes 2026-02-20T21:53:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1516722176 bytes 2026-02-20T21:53:55.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1565069312 bytes 2026-02-20T21:53:55.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1583677440 bytes 2026-02-20T21:53:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1607269376 bytes 2026-02-20T21:53:55.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1663806464 bytes 2026-02-20T21:53:55.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1712929792 bytes 2026-02-20T21:53:55.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1741912064 bytes 2026-02-20T21:53:56.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1779396608 bytes 2026-02-20T21:53:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1838829568 bytes 2026-02-20T21:53:56.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1885976576 bytes 2026-02-20T21:53:56.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1981033472 bytes 2026-02-20T21:53:56.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2024468480 bytes 2026-02-20T21:53:56.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2087410688 bytes 2026-02-20T21:53:56.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2117177344 bytes 2026-02-20T21:53:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2144747520 bytes 2026-02-20T21:53:56.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2156931072 bytes 2026-02-20T21:53:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2231085056 bytes 2026-02-20T21:53:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2263155712 bytes 2026-02-20T21:53:56.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2326912000 bytes 2026-02-20T21:53:56.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2400119808 bytes 2026-02-20T21:53:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2506577920 bytes 2026-02-20T21:53:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2533453824 bytes 2026-02-20T21:53:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2606125056 bytes 2026-02-20T21:53:56.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2673418240 bytes 2026-02-20T21:53:56.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2683933696 bytes 2026-02-20T21:53:56.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2723422208 bytes 2026-02-20T21:53:57.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2757578752 bytes 2026-02-20T21:53:57.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2780351488 bytes 2026-02-20T21:53:57.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2802221056 bytes 2026-02-20T21:53:57.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2814473216 bytes 2026-02-20T21:53:57.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2824838144 bytes 2026-02-20T21:53:57.207 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2868478976 bytes 2026-02-20T21:53:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2896884736 bytes 2026-02-20T21:53:57.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2931628032 bytes 2026-02-20T21:53:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2975024128 bytes 2026-02-20T21:53:57.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3006559232 bytes 2026-02-20T21:53:57.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3067049984 bytes 2026-02-20T21:53:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3095192576 bytes 2026-02-20T21:53:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3154584576 bytes 2026-02-20T21:53:57.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3170012160 bytes 2026-02-20T21:53:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3231918080 bytes 2026-02-20T21:53:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3261224960 bytes 2026-02-20T21:53:57.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3271674880 bytes 2026-02-20T21:53:57.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3355965440 bytes 2026-02-20T21:53:57.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3382932480 bytes 2026-02-20T21:53:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3406612480 bytes 2026-02-20T21:53:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3450587136 bytes 2026-02-20T21:53:57.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3471244288 bytes 2026-02-20T21:53:58.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3556262912 bytes 2026-02-20T21:53:58.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3559678976 bytes 2026-02-20T21:53:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3594677248 bytes 2026-02-20T21:53:58.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3611055104 bytes 2026-02-20T21:53:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3677680640 bytes 2026-02-20T21:53:58.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3782404096 bytes 2026-02-20T21:53:58.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3804403712 bytes 2026-02-20T21:53:58.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3918969856 bytes 2026-02-20T21:53:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3950301184 bytes 2026-02-20T21:53:58.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3970066432 bytes 2026-02-20T21:53:58.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4032013312 bytes 2026-02-20T21:53:58.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4041449472 bytes 2026-02-20T21:53:58.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4053868544 bytes 2026-02-20T21:53:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4117297152 bytes 2026-02-20T21:53:58.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4168795136 bytes 2026-02-20T21:53:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4220045312 bytes 2026-02-20T21:53:58.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4220673024 bytes 2026-02-20T21:53:58.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4280620032 bytes 2026-02-20T21:53:58.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4331037696 bytes 2026-02-20T21:53:58.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4365292544 bytes 2026-02-20T21:53:59.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4432573440 bytes 2026-02-20T21:53:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4492021760 bytes 2026-02-20T21:53:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4506476544 bytes 2026-02-20T21:53:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4530609152 bytes 2026-02-20T21:53:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4532359168 bytes 2026-02-20T21:53:59.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4568514560 bytes 2026-02-20T21:53:59.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4667054080 bytes 2026-02-20T21:53:59.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4739488768 bytes 2026-02-20T21:53:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4816924672 bytes 2026-02-20T21:53:59.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4864673792 bytes 2026-02-20T21:53:59.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4865309696 bytes 2026-02-20T21:53:59.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4925034496 bytes 2026-02-20T21:53:59.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4955680768 bytes 2026-02-20T21:53:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5006439424 bytes 2026-02-20T21:53:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5044560896 bytes 2026-02-20T21:53:59.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5057775616 bytes 2026-02-20T21:53:59.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5124585472 bytes 2026-02-20T21:53:59.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5137371136 bytes 2026-02-20T21:53:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5168468992 bytes 2026-02-20T21:53:59.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5234857984 bytes 2026-02-20T21:54:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5256927232 bytes 2026-02-20T21:54:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5275975680 bytes 2026-02-20T21:54:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5302323200 bytes 2026-02-20T21:54:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5363351552 bytes 2026-02-20T21:54:00.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5409573888 bytes 2026-02-20T21:54:00.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5412209664 bytes 2026-02-20T21:54:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5433765888 bytes 2026-02-20T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5446279168 bytes 2026-02-20T21:54:00.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5501717504 bytes 2026-02-20T21:54:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5585754112 bytes 2026-02-20T21:54:00.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5648560128 bytes 2026-02-20T21:54:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5661041664 bytes 2026-02-20T21:54:00.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5661664256 bytes 2026-02-20T21:54:00.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5679665152 bytes 2026-02-20T21:54:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5761113088 bytes 2026-02-20T21:54:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5761717248 bytes 2026-02-20T21:54:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5783704576 bytes 2026-02-20T21:54:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5858984960 bytes 2026-02-20T21:54:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5922214912 bytes 2026-02-20T21:54:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5987357696 bytes 2026-02-20T21:54:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6018727936 bytes 2026-02-20T21:54:01.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6090085376 bytes 2026-02-20T21:54:01.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6090677248 bytes 2026-02-20T21:54:01.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6130359296 bytes 2026-02-20T21:54:01.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6131502080 bytes 2026-02-20T21:54:01.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6167129088 bytes 2026-02-20T21:54:01.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6224741376 bytes 2026-02-20T21:54:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6316180480 bytes 2026-02-20T21:54:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6390946816 bytes 2026-02-20T21:54:01.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6450147328 bytes 2026-02-20T21:54:01.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6471353344 bytes 2026-02-20T21:54:01.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6506984448 bytes 2026-02-20T21:54:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6507591680 bytes 2026-02-20T21:54:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6531771392 bytes 2026-02-20T21:54:01.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6592697344 bytes 2026-02-20T21:54:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6593308672 bytes 2026-02-20T21:54:01.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6615474176 bytes 2026-02-20T21:54:01.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6658381824 bytes 2026-02-20T21:54:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6668317696 bytes 2026-02-20T21:54:01.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6703078400 bytes 2026-02-20T21:54:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6735990784 bytes 2026-02-20T21:54:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6793612288 bytes 2026-02-20T21:54:01.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6835154944 bytes 2026-02-20T21:54:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6873652224 bytes 2026-02-20T21:54:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6889053184 bytes 2026-02-20T21:54:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 6911968256 bytes 2026-02-20T21:54:02.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7017843712 bytes 2026-02-20T21:54:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7056904192 bytes 2026-02-20T21:54:02.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7114479616 bytes 2026-02-20T21:54:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7177166848 bytes 2026-02-20T21:54:02.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7207632896 bytes 2026-02-20T21:54:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7244238848 bytes 2026-02-20T21:54:02.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7322357760 bytes 2026-02-20T21:54:02.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7365030912 bytes 2026-02-20T21:54:02.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7370198016 bytes 2026-02-20T21:54:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7417226240 bytes 2026-02-20T21:54:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7423948800 bytes 2026-02-20T21:54:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7456888832 bytes 2026-02-20T21:54:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7506020352 bytes 2026-02-20T21:54:02.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7571664896 bytes 2026-02-20T21:54:02.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7625234432 bytes 2026-02-20T21:54:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7682955264 bytes 2026-02-20T21:54:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7762294784 bytes 2026-02-20T21:54:03.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7765079040 bytes 2026-02-20T21:54:03.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7828389888 bytes 2026-02-20T21:54:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7887758336 bytes 2026-02-20T21:54:03.125 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:54:03.126 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:54:03.126 INFO:tasks.rbd_mirror_thrash:waiting for 5.339182933960114 secs before reviving daemons 2026-02-20T21:54:03.126 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T21:54:03.124+0000 7f04a5d15640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 25434) UID: 1000 2026-02-20T21:54:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 7927122944 bytes 2026-02-20T21:54:03.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 7945976832 bytes 2026-02-20T21:54:03.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 7946596352 bytes 2026-02-20T21:54:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 7968995328 bytes 2026-02-20T21:54:03.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8016765952 bytes 2026-02-20T21:54:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8060971008 bytes 2026-02-20T21:54:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8098902016 bytes 2026-02-20T21:54:03.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8135504896 bytes 2026-02-20T21:54:03.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8203294720 bytes 2026-02-20T21:54:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8253709312 bytes 2026-02-20T21:54:03.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8343744512 bytes 2026-02-20T21:54:03.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8384328704 bytes 2026-02-20T21:54:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:54:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:54:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:54:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:54:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_38 2026-02-20T21:54:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 snapshot 128M 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_38 128M 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:03.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T21:54:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:54:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:54:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:54:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:03.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_38 2026-02-20T21:54:03.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:54:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:54:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_38 snapshot 2026-02-20T21:54:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T21:54:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:03.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_38 snapshot 2026-02-20T21:54:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:54:04.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_38 2026-02-20T21:54:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_38': 2026-02-20T21:54:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:54:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:54:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:54:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 144678c87625 2026-02-20T21:54:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.144678c87625 2026-02-20T21:54:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:03 2026 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:03 2026 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:03 2026 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: f0d4e737-438c-4247-8539-7b6130b2be4d 2026-02-20T21:54:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:54:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:54:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_38 2026-02-20T21:54:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T21:54:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:54:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=10 2026-02-20T21:54:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:54:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:54:04.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 25967616 bytes 2026-02-20T21:54:04.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 39556096 bytes 2026-02-20T21:54:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 77783040 bytes 2026-02-20T21:54:05.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 151343104 bytes 2026-02-20T21:54:05.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 181888000 bytes 2026-02-20T21:54:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 196990976 bytes 2026-02-20T21:54:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 243445760 bytes 2026-02-20T21:54:05.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 280814592 bytes 2026-02-20T21:54:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 281438208 bytes 2026-02-20T21:54:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 324719616 bytes 2026-02-20T21:54:05.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 325300224 bytes 2026-02-20T21:54:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 346577920 bytes 2026-02-20T21:54:05.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 383525888 bytes 2026-02-20T21:54:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 459717632 bytes 2026-02-20T21:54:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 464002048 bytes 2026-02-20T21:54:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 500151296 bytes 2026-02-20T21:54:05.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 555869184 bytes 2026-02-20T21:54:05.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 591553536 bytes 2026-02-20T21:54:05.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 719142912 bytes 2026-02-20T21:54:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 719752192 bytes 2026-02-20T21:54:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 744269824 bytes 2026-02-20T21:54:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 772829184 bytes 2026-02-20T21:54:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 782053376 bytes 2026-02-20T21:54:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 793275392 bytes 2026-02-20T21:54:05.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 816667648 bytes 2026-02-20T21:54:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 852746240 bytes 2026-02-20T21:54:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 853350400 bytes 2026-02-20T21:54:06.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 900830208 bytes 2026-02-20T21:54:06.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 993989632 bytes 2026-02-20T21:54:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1029680128 bytes 2026-02-20T21:54:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1135968256 bytes 2026-02-20T21:54:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1239994368 bytes 2026-02-20T21:54:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1263943680 bytes 2026-02-20T21:54:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1291762688 bytes 2026-02-20T21:54:06.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1311388672 bytes 2026-02-20T21:54:06.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1333856256 bytes 2026-02-20T21:54:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1387814912 bytes 2026-02-20T21:54:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1409726464 bytes 2026-02-20T21:54:06.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1473825792 bytes 2026-02-20T21:54:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1497312256 bytes 2026-02-20T21:54:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1524215808 bytes 2026-02-20T21:54:06.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1610396672 bytes 2026-02-20T21:54:06.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1634135040 bytes 2026-02-20T21:54:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1649285120 bytes 2026-02-20T21:54:06.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1685871616 bytes 2026-02-20T21:54:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1737596928 bytes 2026-02-20T21:54:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1789867008 bytes 2026-02-20T21:54:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1818049536 bytes 2026-02-20T21:54:07.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1907445760 bytes 2026-02-20T21:54:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1927274496 bytes 2026-02-20T21:54:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2000145408 bytes 2026-02-20T21:54:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2068209664 bytes 2026-02-20T21:54:07.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2093952000 bytes 2026-02-20T21:54:07.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2148629504 bytes 2026-02-20T21:54:07.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2207650816 bytes 2026-02-20T21:54:07.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2276078592 bytes 2026-02-20T21:54:07.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2322834432 bytes 2026-02-20T21:54:07.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2351509504 bytes 2026-02-20T21:54:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2376186880 bytes 2026-02-20T21:54:07.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2441035776 bytes 2026-02-20T21:54:07.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2447298560 bytes 2026-02-20T21:54:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2454958080 bytes 2026-02-20T21:54:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2479946752 bytes 2026-02-20T21:54:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2513175552 bytes 2026-02-20T21:54:07.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2571981824 bytes 2026-02-20T21:54:08.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2619254784 bytes 2026-02-20T21:54:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2662398976 bytes 2026-02-20T21:54:08.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2707473408 bytes 2026-02-20T21:54:08.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2771897344 bytes 2026-02-20T21:54:08.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2808777728 bytes 2026-02-20T21:54:08.465 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:54:08.466 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:54:08.466 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:54:08.466 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:54:08.466 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:54:08.468 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:54:08.468 INFO:tasks.rbd_mirror_thrash:waiting for 67.95123430348295 secs before thrashing 2026-02-20T21:54:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2855860224 bytes 2026-02-20T21:54:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2888110080 bytes 2026-02-20T21:54:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2924304384 bytes 2026-02-20T21:54:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2949296128 bytes 2026-02-20T21:54:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3016564736 bytes 2026-02-20T21:54:08.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3066204160 bytes 2026-02-20T21:54:08.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3108715520 bytes 2026-02-20T21:54:08.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3135566848 bytes 2026-02-20T21:54:08.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3175071744 bytes 2026-02-20T21:54:08.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3175703552 bytes 2026-02-20T21:54:08.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3199127552 bytes 2026-02-20T21:54:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3282417664 bytes 2026-02-20T21:54:09.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3301572608 bytes 2026-02-20T21:54:09.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3351496704 bytes 2026-02-20T21:54:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3429312512 bytes 2026-02-20T21:54:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3466895360 bytes 2026-02-20T21:54:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3524336640 bytes 2026-02-20T21:54:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3597047808 bytes 2026-02-20T21:54:09.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3621544960 bytes 2026-02-20T21:54:09.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3691196416 bytes 2026-02-20T21:54:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3736466432 bytes 2026-02-20T21:54:09.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3780445184 bytes 2026-02-20T21:54:09.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3784297472 bytes 2026-02-20T21:54:09.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3817468928 bytes 2026-02-20T21:54:09.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3845603328 bytes 2026-02-20T21:54:09.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3914897408 bytes 2026-02-20T21:54:09.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3915521024 bytes 2026-02-20T21:54:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 3946175488 bytes 2026-02-20T21:54:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 3982286848 bytes 2026-02-20T21:54:09.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 3998253056 bytes 2026-02-20T21:54:09.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4037065728 bytes 2026-02-20T21:54:09.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4065376256 bytes 2026-02-20T21:54:09.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4121973760 bytes 2026-02-20T21:54:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4176567296 bytes 2026-02-20T21:54:10.100 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4240900096 bytes 2026-02-20T21:54:10.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4307394560 bytes 2026-02-20T21:54:10.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4374964224 bytes 2026-02-20T21:54:10.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4444913664 bytes 2026-02-20T21:54:10.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4459729920 bytes 2026-02-20T21:54:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4493691904 bytes 2026-02-20T21:54:10.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4505171968 bytes 2026-02-20T21:54:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4505792512 bytes 2026-02-20T21:54:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4512293888 bytes 2026-02-20T21:54:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4542579712 bytes 2026-02-20T21:54:10.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4546601984 bytes 2026-02-20T21:54:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4623606784 bytes 2026-02-20T21:54:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4624256000 bytes 2026-02-20T21:54:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4648725504 bytes 2026-02-20T21:54:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4708794368 bytes 2026-02-20T21:54:10.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4747573248 bytes 2026-02-20T21:54:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4774729728 bytes 2026-02-20T21:54:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 4807004160 bytes 2026-02-20T21:54:10.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 4885964800 bytes 2026-02-20T21:54:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 4897323008 bytes 2026-02-20T21:54:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 4952262656 bytes 2026-02-20T21:54:11.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 4978525184 bytes 2026-02-20T21:54:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5034330112 bytes 2026-02-20T21:54:11.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5054170112 bytes 2026-02-20T21:54:11.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5072194560 bytes 2026-02-20T21:54:11.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5114555392 bytes 2026-02-20T21:54:11.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5202353152 bytes 2026-02-20T21:54:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5225667584 bytes 2026-02-20T21:54:11.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5238043648 bytes 2026-02-20T21:54:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5282601984 bytes 2026-02-20T21:54:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5297225728 bytes 2026-02-20T21:54:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5337235456 bytes 2026-02-20T21:54:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5337844736 bytes 2026-02-20T21:54:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5396389888 bytes 2026-02-20T21:54:11.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5413583872 bytes 2026-02-20T21:54:11.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5537453056 bytes 2026-02-20T21:54:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5631759360 bytes 2026-02-20T21:54:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5674166272 bytes 2026-02-20T21:54:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5717285888 bytes 2026-02-20T21:54:11.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5777649664 bytes 2026-02-20T21:54:11.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5778265088 bytes 2026-02-20T21:54:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5778884608 bytes 2026-02-20T21:54:12.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5811657728 bytes 2026-02-20T21:54:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5847571456 bytes 2026-02-20T21:54:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 5859722240 bytes 2026-02-20T21:54:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 5860345856 bytes 2026-02-20T21:54:12.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 5895294976 bytes 2026-02-20T21:54:12.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 5924175872 bytes 2026-02-20T21:54:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 5966062592 bytes 2026-02-20T21:54:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6024855552 bytes 2026-02-20T21:54:12.361 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:54:12.356+0000 7fa5e2658640 -1 reset not still connected to 0x55b9054ee340 2026-02-20T21:54:12.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6032424960 bytes 2026-02-20T21:54:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6062374912 bytes 2026-02-20T21:54:12.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6109214720 bytes 2026-02-20T21:54:12.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6146989056 bytes 2026-02-20T21:54:12.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6233798656 bytes 2026-02-20T21:54:12.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6250546176 bytes 2026-02-20T21:54:12.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6285721600 bytes 2026-02-20T21:54:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6338749440 bytes 2026-02-20T21:54:12.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6382257152 bytes 2026-02-20T21:54:12.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6402153472 bytes 2026-02-20T21:54:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6479237120 bytes 2026-02-20T21:54:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6534591488 bytes 2026-02-20T21:54:12.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6560232448 bytes 2026-02-20T21:54:13.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6599038976 bytes 2026-02-20T21:54:13.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6626156544 bytes 2026-02-20T21:54:13.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 6706318336 bytes 2026-02-20T21:54:13.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 6751308800 bytes 2026-02-20T21:54:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 6774695936 bytes 2026-02-20T21:54:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 6819167232 bytes 2026-02-20T21:54:13.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 6890107904 bytes 2026-02-20T21:54:13.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 6922782720 bytes 2026-02-20T21:54:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 6923424768 bytes 2026-02-20T21:54:13.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 6992855040 bytes 2026-02-20T21:54:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7061814272 bytes 2026-02-20T21:54:13.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7137762304 bytes 2026-02-20T21:54:13.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7171209216 bytes 2026-02-20T21:54:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7216357376 bytes 2026-02-20T21:54:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7254814720 bytes 2026-02-20T21:54:13.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7295315968 bytes 2026-02-20T21:54:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7391637504 bytes 2026-02-20T21:54:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7433501696 bytes 2026-02-20T21:54:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7457040384 bytes 2026-02-20T21:54:13.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7483244544 bytes 2026-02-20T21:54:14.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7524217856 bytes 2026-02-20T21:54:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7600445440 bytes 2026-02-20T21:54:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7621230592 bytes 2026-02-20T21:54:14.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7659011072 bytes 2026-02-20T21:54:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7696520192 bytes 2026-02-20T21:54:14.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7723355136 bytes 2026-02-20T21:54:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 7782428672 bytes 2026-02-20T21:54:14.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 7846690816 bytes 2026-02-20T21:54:14.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 7847298048 bytes 2026-02-20T21:54:14.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 7902887936 bytes 2026-02-20T21:54:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 7950576640 bytes 2026-02-20T21:54:14.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 7984171008 bytes 2026-02-20T21:54:14.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8035535872 bytes 2026-02-20T21:54:14.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8141413376 bytes 2026-02-20T21:54:14.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8210082816 bytes 2026-02-20T21:54:14.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:54:14.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:54:14.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:54:14.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_39 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 snapshot 128M 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_39 128M 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:54:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:14.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_39 2026-02-20T21:54:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:54:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:54:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_39 snapshot 2026-02-20T21:54:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T21:54:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_39 snapshot 2026-02-20T21:54:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:54:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_39 2026-02-20T21:54:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_39': 2026-02-20T21:54:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:54:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:54:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:54:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 145b833191e7 2026-02-20T21:54:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.145b833191e7 2026-02-20T21:54:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:54:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:54:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:54:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:54:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:14 2026 2026-02-20T21:54:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:14 2026 2026-02-20T21:54:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:14 2026 2026-02-20T21:54:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:54:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:54:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 876e1da9-8a7e-447b-81f0-6c8df3b23c0c 2026-02-20T21:54:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:54:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:54:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_39 2026-02-20T21:54:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T21:54:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:54:15.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=14 2026-02-20T21:54:15.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:54:15.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:54:15.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 22916096 bytes 2026-02-20T21:54:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 72507392 bytes 2026-02-20T21:54:15.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 135046144 bytes 2026-02-20T21:54:15.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 156274688 bytes 2026-02-20T21:54:16.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 198095872 bytes 2026-02-20T21:54:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 212260864 bytes 2026-02-20T21:54:16.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 328934400 bytes 2026-02-20T21:54:16.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 380688384 bytes 2026-02-20T21:54:16.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 408136704 bytes 2026-02-20T21:54:16.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 429156352 bytes 2026-02-20T21:54:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 448621568 bytes 2026-02-20T21:54:16.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 470992896 bytes 2026-02-20T21:54:16.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 513842176 bytes 2026-02-20T21:54:16.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 533580800 bytes 2026-02-20T21:54:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 575619072 bytes 2026-02-20T21:54:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 608184320 bytes 2026-02-20T21:54:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 637151232 bytes 2026-02-20T21:54:16.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 689412096 bytes 2026-02-20T21:54:16.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 780796928 bytes 2026-02-20T21:54:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 793780224 bytes 2026-02-20T21:54:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 823864320 bytes 2026-02-20T21:54:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 871784448 bytes 2026-02-20T21:54:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 892755968 bytes 2026-02-20T21:54:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 895110144 bytes 2026-02-20T21:54:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 946563072 bytes 2026-02-20T21:54:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 995403776 bytes 2026-02-20T21:54:17.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1014044672 bytes 2026-02-20T21:54:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1041280000 bytes 2026-02-20T21:54:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1136754688 bytes 2026-02-20T21:54:17.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1184643072 bytes 2026-02-20T21:54:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1295598592 bytes 2026-02-20T21:54:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1343090688 bytes 2026-02-20T21:54:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1345931264 bytes 2026-02-20T21:54:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1364593664 bytes 2026-02-20T21:54:17.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1394419712 bytes 2026-02-20T21:54:17.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1433235456 bytes 2026-02-20T21:54:17.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1462610944 bytes 2026-02-20T21:54:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1507974144 bytes 2026-02-20T21:54:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1528065024 bytes 2026-02-20T21:54:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1613968384 bytes 2026-02-20T21:54:17.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1621956608 bytes 2026-02-20T21:54:17.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1633824768 bytes 2026-02-20T21:54:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1745230848 bytes 2026-02-20T21:54:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1834767360 bytes 2026-02-20T21:54:17.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1860019200 bytes 2026-02-20T21:54:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1934299136 bytes 2026-02-20T21:54:18.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2010192896 bytes 2026-02-20T21:54:18.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2081150976 bytes 2026-02-20T21:54:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2132003840 bytes 2026-02-20T21:54:18.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2163863552 bytes 2026-02-20T21:54:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2184853504 bytes 2026-02-20T21:54:18.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2218925056 bytes 2026-02-20T21:54:18.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2219536384 bytes 2026-02-20T21:54:18.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2231534592 bytes 2026-02-20T21:54:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2253838336 bytes 2026-02-20T21:54:18.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2291085312 bytes 2026-02-20T21:54:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2367438848 bytes 2026-02-20T21:54:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2389582848 bytes 2026-02-20T21:54:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2407085056 bytes 2026-02-20T21:54:18.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2463313920 bytes 2026-02-20T21:54:18.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2463919104 bytes 2026-02-20T21:54:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2500756480 bytes 2026-02-20T21:54:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2530242560 bytes 2026-02-20T21:54:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2565434368 bytes 2026-02-20T21:54:18.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2608091136 bytes 2026-02-20T21:54:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2624168960 bytes 2026-02-20T21:54:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2664420352 bytes 2026-02-20T21:54:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2761252864 bytes 2026-02-20T21:54:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2832250880 bytes 2026-02-20T21:54:19.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2851933184 bytes 2026-02-20T21:54:19.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2863104000 bytes 2026-02-20T21:54:19.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2887290880 bytes 2026-02-20T21:54:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2915843072 bytes 2026-02-20T21:54:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2969079808 bytes 2026-02-20T21:54:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 2982802432 bytes 2026-02-20T21:54:19.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3020182528 bytes 2026-02-20T21:54:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3054176256 bytes 2026-02-20T21:54:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3131484160 bytes 2026-02-20T21:54:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3206236160 bytes 2026-02-20T21:54:19.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3246050304 bytes 2026-02-20T21:54:19.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3295917056 bytes 2026-02-20T21:54:19.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3313363968 bytes 2026-02-20T21:54:19.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3337057280 bytes 2026-02-20T21:54:19.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3354162176 bytes 2026-02-20T21:54:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3394755584 bytes 2026-02-20T21:54:19.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3490422784 bytes 2026-02-20T21:54:19.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3555846144 bytes 2026-02-20T21:54:20.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3585141760 bytes 2026-02-20T21:54:20.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3592551424 bytes 2026-02-20T21:54:20.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3651696640 bytes 2026-02-20T21:54:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3666810880 bytes 2026-02-20T21:54:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3686943744 bytes 2026-02-20T21:54:20.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3768283136 bytes 2026-02-20T21:54:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3810172928 bytes 2026-02-20T21:54:20.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3854729216 bytes 2026-02-20T21:54:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3910745088 bytes 2026-02-20T21:54:20.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3921713152 bytes 2026-02-20T21:54:20.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4009789440 bytes 2026-02-20T21:54:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4040551424 bytes 2026-02-20T21:54:20.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4092598272 bytes 2026-02-20T21:54:20.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4114706432 bytes 2026-02-20T21:54:20.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4222462976 bytes 2026-02-20T21:54:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4271215616 bytes 2026-02-20T21:54:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4326475776 bytes 2026-02-20T21:54:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4400808960 bytes 2026-02-20T21:54:20.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4466124800 bytes 2026-02-20T21:54:20.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4480351232 bytes 2026-02-20T21:54:21.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4540649472 bytes 2026-02-20T21:54:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4547291136 bytes 2026-02-20T21:54:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4635795456 bytes 2026-02-20T21:54:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4697459712 bytes 2026-02-20T21:54:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4766638080 bytes 2026-02-20T21:54:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4834710528 bytes 2026-02-20T21:54:21.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4918088704 bytes 2026-02-20T21:54:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4944974848 bytes 2026-02-20T21:54:21.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4983154688 bytes 2026-02-20T21:54:21.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5021222912 bytes 2026-02-20T21:54:21.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5056724992 bytes 2026-02-20T21:54:21.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5157640192 bytes 2026-02-20T21:54:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5198669824 bytes 2026-02-20T21:54:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5253820416 bytes 2026-02-20T21:54:21.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5276220416 bytes 2026-02-20T21:54:21.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5314001920 bytes 2026-02-20T21:54:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5351297024 bytes 2026-02-20T21:54:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5395720192 bytes 2026-02-20T21:54:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5471247360 bytes 2026-02-20T21:54:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5524355072 bytes 2026-02-20T21:54:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5592042496 bytes 2026-02-20T21:54:22.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5592639488 bytes 2026-02-20T21:54:22.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5631227904 bytes 2026-02-20T21:54:22.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5669951488 bytes 2026-02-20T21:54:22.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5718574080 bytes 2026-02-20T21:54:22.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5782860800 bytes 2026-02-20T21:54:22.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5796939776 bytes 2026-02-20T21:54:22.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5812459520 bytes 2026-02-20T21:54:22.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5902446592 bytes 2026-02-20T21:54:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5948180480 bytes 2026-02-20T21:54:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5984552960 bytes 2026-02-20T21:54:22.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6085586944 bytes 2026-02-20T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6132042752 bytes 2026-02-20T21:54:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6191329280 bytes 2026-02-20T21:54:22.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6237070336 bytes 2026-02-20T21:54:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6265135104 bytes 2026-02-20T21:54:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6283228160 bytes 2026-02-20T21:54:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6334169088 bytes 2026-02-20T21:54:22.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6438166528 bytes 2026-02-20T21:54:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6457279488 bytes 2026-02-20T21:54:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6484882432 bytes 2026-02-20T21:54:23.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6532141056 bytes 2026-02-20T21:54:23.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6543211520 bytes 2026-02-20T21:54:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6596701184 bytes 2026-02-20T21:54:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6631270400 bytes 2026-02-20T21:54:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6658220032 bytes 2026-02-20T21:54:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6698411008 bytes 2026-02-20T21:54:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6733860864 bytes 2026-02-20T21:54:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6755291136 bytes 2026-02-20T21:54:23.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6780796928 bytes 2026-02-20T21:54:23.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6870734848 bytes 2026-02-20T21:54:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6890327040 bytes 2026-02-20T21:54:23.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6995321856 bytes 2026-02-20T21:54:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7049467904 bytes 2026-02-20T21:54:23.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7093781504 bytes 2026-02-20T21:54:23.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7216378880 bytes 2026-02-20T21:54:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7262306304 bytes 2026-02-20T21:54:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7279941632 bytes 2026-02-20T21:54:23.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7302946816 bytes 2026-02-20T21:54:23.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7379492864 bytes 2026-02-20T21:54:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7410208768 bytes 2026-02-20T21:54:24.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7456685056 bytes 2026-02-20T21:54:24.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7476666368 bytes 2026-02-20T21:54:24.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7522573312 bytes 2026-02-20T21:54:24.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7569732608 bytes 2026-02-20T21:54:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7628109824 bytes 2026-02-20T21:54:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7679637504 bytes 2026-02-20T21:54:24.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7714128896 bytes 2026-02-20T21:54:24.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7792921600 bytes 2026-02-20T21:54:24.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7850667008 bytes 2026-02-20T21:54:24.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7870410752 bytes 2026-02-20T21:54:24.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7892894720 bytes 2026-02-20T21:54:24.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7942047744 bytes 2026-02-20T21:54:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7963084800 bytes 2026-02-20T21:54:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 8010438656 bytes 2026-02-20T21:54:24.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 8040136704 bytes 2026-02-20T21:54:25.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 8063966208 bytes 2026-02-20T21:54:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 8086366208 bytes 2026-02-20T21:54:27.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 8129075200 bytes 2026-02-20T21:54:27.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8184139776 bytes 2026-02-20T21:54:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8196476928 bytes 2026-02-20T21:54:27.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8256098304 bytes 2026-02-20T21:54:27.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8342127616 bytes 2026-02-20T21:54:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8352362496 bytes 2026-02-20T21:54:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8447839232 bytes 2026-02-20T21:54:28.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8471738368 bytes 2026-02-20T21:54:28.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8478057472 bytes 2026-02-20T21:54:28.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8505737216 bytes 2026-02-20T21:54:28.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8615332864 bytes 2026-02-20T21:54:28.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8652066816 bytes 2026-02-20T21:54:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8735532032 bytes 2026-02-20T21:54:28.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8782699520 bytes 2026-02-20T21:54:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8790256640 bytes 2026-02-20T21:54:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8880332800 bytes 2026-02-20T21:54:28.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8919497728 bytes 2026-02-20T21:54:28.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8979903488 bytes 2026-02-20T21:54:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 9043182592 bytes 2026-02-20T21:54:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 9053145088 bytes 2026-02-20T21:54:28.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 9099942912 bytes 2026-02-20T21:54:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 9166991360 bytes 2026-02-20T21:54:28.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 9316795392 bytes 2026-02-20T21:54:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 9341889536 bytes 2026-02-20T21:54:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9357231104 bytes 2026-02-20T21:54:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9418645504 bytes 2026-02-20T21:54:29.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9445735424 bytes 2026-02-20T21:54:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9527025664 bytes 2026-02-20T21:54:29.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9557549056 bytes 2026-02-20T21:54:29.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9625230336 bytes 2026-02-20T21:54:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9676926976 bytes 2026-02-20T21:54:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9770382336 bytes 2026-02-20T21:54:29.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9837036544 bytes 2026-02-20T21:54:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9862795264 bytes 2026-02-20T21:54:29.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9896887296 bytes 2026-02-20T21:54:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9950054400 bytes 2026-02-20T21:54:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9978445824 bytes 2026-02-20T21:54:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 10023555072 bytes 2026-02-20T21:54:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 10070490112 bytes 2026-02-20T21:54:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 10095494144 bytes 2026-02-20T21:54:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:54:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:54:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:54:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:54:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_40 2026-02-20T21:54:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 snapshot 128M 2026-02-20T21:54:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T21:54:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:54:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_40 128M 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_40 2026-02-20T21:54:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:54:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:54:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_40 snapshot 2026-02-20T21:54:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T21:54:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_40 snapshot 2026-02-20T21:54:30.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:54:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_40 2026-02-20T21:54:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_40': 2026-02-20T21:54:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:54:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:54:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:54:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 146724d55ddd 2026-02-20T21:54:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.146724d55ddd 2026-02-20T21:54:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:54:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:54:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:54:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:54:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:29 2026 2026-02-20T21:54:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:29 2026 2026-02-20T21:54:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:29 2026 2026-02-20T21:54:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:54:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:54:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: c4c8b7cd-d483-42c9-9489-fa930aaeae6b 2026-02-20T21:54:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:54:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:54:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_40 2026-02-20T21:54:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T21:54:30.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:54:30.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=10 2026-02-20T21:54:30.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:54:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:54:30.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 57843712 bytes 2026-02-20T21:54:30.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 104535040 bytes 2026-02-20T21:54:30.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 224489472 bytes 2026-02-20T21:54:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 259722240 bytes 2026-02-20T21:54:31.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 335970304 bytes 2026-02-20T21:54:31.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 361479168 bytes 2026-02-20T21:54:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 415276032 bytes 2026-02-20T21:54:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 480477184 bytes 2026-02-20T21:54:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 503882752 bytes 2026-02-20T21:54:31.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 507482112 bytes 2026-02-20T21:54:31.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 546890752 bytes 2026-02-20T21:54:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 618382336 bytes 2026-02-20T21:54:31.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 670608384 bytes 2026-02-20T21:54:31.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 739693568 bytes 2026-02-20T21:54:31.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 844899328 bytes 2026-02-20T21:54:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 897874944 bytes 2026-02-20T21:54:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 950541312 bytes 2026-02-20T21:54:31.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 996549632 bytes 2026-02-20T21:54:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 1021514752 bytes 2026-02-20T21:54:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1080024064 bytes 2026-02-20T21:54:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1098836992 bytes 2026-02-20T21:54:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1135906816 bytes 2026-02-20T21:54:31.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1136507904 bytes 2026-02-20T21:54:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1148754944 bytes 2026-02-20T21:54:32.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1166686208 bytes 2026-02-20T21:54:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1190628352 bytes 2026-02-20T21:54:32.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1240407040 bytes 2026-02-20T21:54:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1299050496 bytes 2026-02-20T21:54:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1322360832 bytes 2026-02-20T21:54:32.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1364657152 bytes 2026-02-20T21:54:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1382950912 bytes 2026-02-20T21:54:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1467361280 bytes 2026-02-20T21:54:32.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1477781504 bytes 2026-02-20T21:54:32.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1522984960 bytes 2026-02-20T21:54:32.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1524932608 bytes 2026-02-20T21:54:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1534198784 bytes 2026-02-20T21:54:32.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1575881728 bytes 2026-02-20T21:54:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1599462400 bytes 2026-02-20T21:54:32.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1625744384 bytes 2026-02-20T21:54:32.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1634339840 bytes 2026-02-20T21:54:32.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1706505216 bytes 2026-02-20T21:54:32.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1732586496 bytes 2026-02-20T21:54:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1791251456 bytes 2026-02-20T21:54:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1828330496 bytes 2026-02-20T21:54:32.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1852666880 bytes 2026-02-20T21:54:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1887349760 bytes 2026-02-20T21:54:33.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1891039232 bytes 2026-02-20T21:54:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1948418048 bytes 2026-02-20T21:54:33.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2021649408 bytes 2026-02-20T21:54:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2079472640 bytes 2026-02-20T21:54:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2101489664 bytes 2026-02-20T21:54:33.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2178462720 bytes 2026-02-20T21:54:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2314487808 bytes 2026-02-20T21:54:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2430054400 bytes 2026-02-20T21:54:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2446343168 bytes 2026-02-20T21:54:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2466328576 bytes 2026-02-20T21:54:33.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2495551488 bytes 2026-02-20T21:54:33.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2584306688 bytes 2026-02-20T21:54:33.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2585606144 bytes 2026-02-20T21:54:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2615516160 bytes 2026-02-20T21:54:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2631938048 bytes 2026-02-20T21:54:33.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2657424384 bytes 2026-02-20T21:54:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2680408064 bytes 2026-02-20T21:54:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2722911232 bytes 2026-02-20T21:54:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2754665472 bytes 2026-02-20T21:54:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2802200576 bytes 2026-02-20T21:54:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2802806784 bytes 2026-02-20T21:54:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2852614144 bytes 2026-02-20T21:54:34.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2910706688 bytes 2026-02-20T21:54:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2928590848 bytes 2026-02-20T21:54:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2951388160 bytes 2026-02-20T21:54:34.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3035483136 bytes 2026-02-20T21:54:34.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3068191744 bytes 2026-02-20T21:54:34.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3097196544 bytes 2026-02-20T21:54:34.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3135783936 bytes 2026-02-20T21:54:34.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3202119680 bytes 2026-02-20T21:54:34.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3268033536 bytes 2026-02-20T21:54:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3288557568 bytes 2026-02-20T21:54:34.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3349154816 bytes 2026-02-20T21:54:34.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3367453696 bytes 2026-02-20T21:54:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3412475904 bytes 2026-02-20T21:54:34.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3464612864 bytes 2026-02-20T21:54:34.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3475562496 bytes 2026-02-20T21:54:34.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3523716096 bytes 2026-02-20T21:54:34.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3592646656 bytes 2026-02-20T21:54:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3624433664 bytes 2026-02-20T21:54:35.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3652327424 bytes 2026-02-20T21:54:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3659114496 bytes 2026-02-20T21:54:35.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3681697792 bytes 2026-02-20T21:54:35.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3782458368 bytes 2026-02-20T21:54:35.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3884193792 bytes 2026-02-20T21:54:35.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3943773184 bytes 2026-02-20T21:54:35.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4067320832 bytes 2026-02-20T21:54:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4097766400 bytes 2026-02-20T21:54:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4114902016 bytes 2026-02-20T21:54:35.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4137295872 bytes 2026-02-20T21:54:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4244680704 bytes 2026-02-20T21:54:35.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4281276416 bytes 2026-02-20T21:54:35.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4315132928 bytes 2026-02-20T21:54:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4331827200 bytes 2026-02-20T21:54:35.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4356707328 bytes 2026-02-20T21:54:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4421748736 bytes 2026-02-20T21:54:35.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4495349760 bytes 2026-02-20T21:54:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4578491392 bytes 2026-02-20T21:54:36.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4611545088 bytes 2026-02-20T21:54:36.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4654584832 bytes 2026-02-20T21:54:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4679544832 bytes 2026-02-20T21:54:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4718404608 bytes 2026-02-20T21:54:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4735740928 bytes 2026-02-20T21:54:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4801471488 bytes 2026-02-20T21:54:36.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4903584768 bytes 2026-02-20T21:54:36.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4960097280 bytes 2026-02-20T21:54:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5014408192 bytes 2026-02-20T21:54:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5055094784 bytes 2026-02-20T21:54:36.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5115385856 bytes 2026-02-20T21:54:36.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5146985472 bytes 2026-02-20T21:54:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5187415040 bytes 2026-02-20T21:54:36.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5204227072 bytes 2026-02-20T21:54:36.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5240617984 bytes 2026-02-20T21:54:36.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5264145408 bytes 2026-02-20T21:54:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5299339264 bytes 2026-02-20T21:54:36.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5310866432 bytes 2026-02-20T21:54:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5337367552 bytes 2026-02-20T21:54:36.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5385133056 bytes 2026-02-20T21:54:36.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5445249024 bytes 2026-02-20T21:54:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5528213504 bytes 2026-02-20T21:54:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5543795712 bytes 2026-02-20T21:54:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5546857472 bytes 2026-02-20T21:54:37.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5591370752 bytes 2026-02-20T21:54:37.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5616088064 bytes 2026-02-20T21:54:37.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5633778688 bytes 2026-02-20T21:54:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5698483200 bytes 2026-02-20T21:54:37.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5796367360 bytes 2026-02-20T21:54:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5831386112 bytes 2026-02-20T21:54:37.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5908978688 bytes 2026-02-20T21:54:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5943376896 bytes 2026-02-20T21:54:37.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5958878208 bytes 2026-02-20T21:54:37.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6025978880 bytes 2026-02-20T21:54:37.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6072022016 bytes 2026-02-20T21:54:37.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6093424640 bytes 2026-02-20T21:54:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6111315968 bytes 2026-02-20T21:54:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6177404928 bytes 2026-02-20T21:54:37.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6204370944 bytes 2026-02-20T21:54:37.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6274706432 bytes 2026-02-20T21:54:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6296561664 bytes 2026-02-20T21:54:38.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6334199808 bytes 2026-02-20T21:54:38.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6334786560 bytes 2026-02-20T21:54:38.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6353771520 bytes 2026-02-20T21:54:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6413188096 bytes 2026-02-20T21:54:38.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6413775872 bytes 2026-02-20T21:54:38.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6462671872 bytes 2026-02-20T21:54:38.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6545453056 bytes 2026-02-20T21:54:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6578636800 bytes 2026-02-20T21:54:38.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6579212288 bytes 2026-02-20T21:54:38.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6638621696 bytes 2026-02-20T21:54:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6712018944 bytes 2026-02-20T21:54:38.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6770063360 bytes 2026-02-20T21:54:38.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6814604288 bytes 2026-02-20T21:54:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6841441280 bytes 2026-02-20T21:54:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6892404736 bytes 2026-02-20T21:54:38.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6903582720 bytes 2026-02-20T21:54:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6980313088 bytes 2026-02-20T21:54:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7040060416 bytes 2026-02-20T21:54:39.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7103716352 bytes 2026-02-20T21:54:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7123433472 bytes 2026-02-20T21:54:39.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7171076096 bytes 2026-02-20T21:54:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7204741120 bytes 2026-02-20T21:54:39.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7224704000 bytes 2026-02-20T21:54:39.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7266891776 bytes 2026-02-20T21:54:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7279788032 bytes 2026-02-20T21:54:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7311090688 bytes 2026-02-20T21:54:39.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7338330112 bytes 2026-02-20T21:54:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7389589504 bytes 2026-02-20T21:54:39.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7428550656 bytes 2026-02-20T21:54:39.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7454437376 bytes 2026-02-20T21:54:39.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7477990400 bytes 2026-02-20T21:54:39.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7542633472 bytes 2026-02-20T21:54:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7597506560 bytes 2026-02-20T21:54:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7657036800 bytes 2026-02-20T21:54:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7723032576 bytes 2026-02-20T21:54:39.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7770029056 bytes 2026-02-20T21:54:39.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7784828928 bytes 2026-02-20T21:54:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7792249856 bytes 2026-02-20T21:54:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7861781504 bytes 2026-02-20T21:54:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7962775552 bytes 2026-02-20T21:54:40.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7982418944 bytes 2026-02-20T21:54:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8037378048 bytes 2026-02-20T21:54:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8109449216 bytes 2026-02-20T21:54:40.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8141811712 bytes 2026-02-20T21:54:40.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8142438400 bytes 2026-02-20T21:54:40.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8181163008 bytes 2026-02-20T21:54:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8195687424 bytes 2026-02-20T21:54:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8230333440 bytes 2026-02-20T21:54:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8274062336 bytes 2026-02-20T21:54:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8297975808 bytes 2026-02-20T21:54:40.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8298972160 bytes 2026-02-20T21:54:40.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8341692416 bytes 2026-02-20T21:54:40.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8375835648 bytes 2026-02-20T21:54:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8420431872 bytes 2026-02-20T21:54:40.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_41 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 snapshot 128M 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:40.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_41 128M 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:40.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:54:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:54:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:54:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:40.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_41 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_41 snapshot 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_41 snapshot 2026-02-20T21:54:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:54:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_41 2026-02-20T21:54:41.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_41': 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 1473bcd69e9f 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.1473bcd69e9f 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:40 2026 2026-02-20T21:54:41.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:40 2026 2026-02-20T21:54:41.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:40 2026 2026-02-20T21:54:41.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:54:41.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:54:41.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: f3be01dc-d6c3-44a9-9f77-d87df8a565ca 2026-02-20T21:54:41.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:54:41.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:54:41.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_41 2026-02-20T21:54:41.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:41.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:41.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T21:54:41.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:54:41.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T21:54:41.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:54:41.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:54:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 39468032 bytes 2026-02-20T21:54:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 91597824 bytes 2026-02-20T21:54:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 110943232 bytes 2026-02-20T21:54:42.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 140125184 bytes 2026-02-20T21:54:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 181232640 bytes 2026-02-20T21:54:42.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 270077952 bytes 2026-02-20T21:54:42.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 328870912 bytes 2026-02-20T21:54:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 383192064 bytes 2026-02-20T21:54:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 435580928 bytes 2026-02-20T21:54:42.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 436193280 bytes 2026-02-20T21:54:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 449296384 bytes 2026-02-20T21:54:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 476053504 bytes 2026-02-20T21:54:42.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 513162240 bytes 2026-02-20T21:54:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 579803136 bytes 2026-02-20T21:54:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 658077696 bytes 2026-02-20T21:54:42.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 703310848 bytes 2026-02-20T21:54:42.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 783337472 bytes 2026-02-20T21:54:42.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 815323136 bytes 2026-02-20T21:54:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 866229248 bytes 2026-02-20T21:54:43.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 877391872 bytes 2026-02-20T21:54:43.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 908018688 bytes 2026-02-20T21:54:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 987012096 bytes 2026-02-20T21:54:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1017339904 bytes 2026-02-20T21:54:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1090329600 bytes 2026-02-20T21:54:43.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1166320640 bytes 2026-02-20T21:54:43.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1187554304 bytes 2026-02-20T21:54:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1231544320 bytes 2026-02-20T21:54:43.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1232168960 bytes 2026-02-20T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1336803328 bytes 2026-02-20T21:54:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1393713152 bytes 2026-02-20T21:54:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1456182272 bytes 2026-02-20T21:54:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1495578624 bytes 2026-02-20T21:54:43.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1515642880 bytes 2026-02-20T21:54:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1554638848 bytes 2026-02-20T21:54:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1617966080 bytes 2026-02-20T21:54:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1667969024 bytes 2026-02-20T21:54:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1677817856 bytes 2026-02-20T21:54:43.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1730975744 bytes 2026-02-20T21:54:44.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1773334528 bytes 2026-02-20T21:54:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1804182528 bytes 2026-02-20T21:54:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1819908096 bytes 2026-02-20T21:54:44.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1843447808 bytes 2026-02-20T21:54:44.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1855873024 bytes 2026-02-20T21:54:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1877909504 bytes 2026-02-20T21:54:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1899436032 bytes 2026-02-20T21:54:44.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1955298304 bytes 2026-02-20T21:54:44.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2017027072 bytes 2026-02-20T21:54:44.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2040783872 bytes 2026-02-20T21:54:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2073573376 bytes 2026-02-20T21:54:44.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2111657984 bytes 2026-02-20T21:54:44.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2146562048 bytes 2026-02-20T21:54:44.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2223671296 bytes 2026-02-20T21:54:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2275676160 bytes 2026-02-20T21:54:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2314189824 bytes 2026-02-20T21:54:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2354873344 bytes 2026-02-20T21:54:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2383024128 bytes 2026-02-20T21:54:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2457268224 bytes 2026-02-20T21:54:44.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2510623744 bytes 2026-02-20T21:54:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2534345728 bytes 2026-02-20T21:54:45.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2577990656 bytes 2026-02-20T21:54:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2635219968 bytes 2026-02-20T21:54:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2696797184 bytes 2026-02-20T21:54:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2746905600 bytes 2026-02-20T21:54:45.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2832884736 bytes 2026-02-20T21:54:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2894945280 bytes 2026-02-20T21:54:45.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2937398272 bytes 2026-02-20T21:54:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2963145728 bytes 2026-02-20T21:54:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2980388864 bytes 2026-02-20T21:54:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3017748480 bytes 2026-02-20T21:54:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3052872704 bytes 2026-02-20T21:54:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3107365888 bytes 2026-02-20T21:54:45.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3125923840 bytes 2026-02-20T21:54:45.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3153697792 bytes 2026-02-20T21:54:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3167156224 bytes 2026-02-20T21:54:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3280228352 bytes 2026-02-20T21:54:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3338754048 bytes 2026-02-20T21:54:45.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3349960704 bytes 2026-02-20T21:54:45.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3425001472 bytes 2026-02-20T21:54:45.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3492048896 bytes 2026-02-20T21:54:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3527313408 bytes 2026-02-20T21:54:46.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3585056768 bytes 2026-02-20T21:54:46.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3635582976 bytes 2026-02-20T21:54:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3662129152 bytes 2026-02-20T21:54:46.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3707231232 bytes 2026-02-20T21:54:46.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3786290176 bytes 2026-02-20T21:54:46.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3823377408 bytes 2026-02-20T21:54:46.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3857716224 bytes 2026-02-20T21:54:46.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3908220928 bytes 2026-02-20T21:54:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3950389248 bytes 2026-02-20T21:54:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3968146432 bytes 2026-02-20T21:54:46.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4065948672 bytes 2026-02-20T21:54:46.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4180056064 bytes 2026-02-20T21:54:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4262096896 bytes 2026-02-20T21:54:46.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4279852032 bytes 2026-02-20T21:54:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4320630784 bytes 2026-02-20T21:54:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4428073984 bytes 2026-02-20T21:54:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4435207168 bytes 2026-02-20T21:54:46.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4457170944 bytes 2026-02-20T21:54:47.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4507602944 bytes 2026-02-20T21:54:47.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4548878336 bytes 2026-02-20T21:54:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4590628864 bytes 2026-02-20T21:54:47.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4644015104 bytes 2026-02-20T21:54:47.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4684775424 bytes 2026-02-20T21:54:47.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4700253184 bytes 2026-02-20T21:54:47.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4751623168 bytes 2026-02-20T21:54:47.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4777150464 bytes 2026-02-20T21:54:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4816335872 bytes 2026-02-20T21:54:47.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4915909632 bytes 2026-02-20T21:54:47.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4989806592 bytes 2026-02-20T21:54:47.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5046543360 bytes 2026-02-20T21:54:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5146274816 bytes 2026-02-20T21:54:47.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5171100672 bytes 2026-02-20T21:54:47.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5188121600 bytes 2026-02-20T21:54:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:54:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:54:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:54:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:54:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_42 2026-02-20T21:54:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 snapshot 128M 2026-02-20T21:54:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_42 128M 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T21:54:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:54:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:54:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:54:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_42 2026-02-20T21:54:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:54:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:54:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_42 snapshot 2026-02-20T21:54:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:47.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T21:54:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:47.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_42 snapshot 2026-02-20T21:54:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:54:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_42 2026-02-20T21:54:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_42': 2026-02-20T21:54:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 147fb3705829 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.147fb3705829 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:47 2026 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:47 2026 2026-02-20T21:54:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:47 2026 2026-02-20T21:54:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:54:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:54:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: e3fc513d-202f-403b-8d2f-5ae273783233 2026-02-20T21:54:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:54:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:54:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_42 2026-02-20T21:54:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T21:54:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:54:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=7 2026-02-20T21:54:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:54:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:54:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 607232 bytes 2026-02-20T21:54:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 44557312 bytes 2026-02-20T21:54:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 47344640 bytes 2026-02-20T21:54:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 137242624 bytes 2026-02-20T21:54:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 167373824 bytes 2026-02-20T21:54:49.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 219323392 bytes 2026-02-20T21:54:49.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 253257728 bytes 2026-02-20T21:54:49.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 278514688 bytes 2026-02-20T21:54:49.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 326380544 bytes 2026-02-20T21:54:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 348974080 bytes 2026-02-20T21:54:49.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 382097408 bytes 2026-02-20T21:54:49.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 382706688 bytes 2026-02-20T21:54:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 464124928 bytes 2026-02-20T21:54:49.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 492310528 bytes 2026-02-20T21:54:49.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 529266688 bytes 2026-02-20T21:54:49.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 604126208 bytes 2026-02-20T21:54:49.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 634000384 bytes 2026-02-20T21:54:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 634616832 bytes 2026-02-20T21:54:49.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 768706560 bytes 2026-02-20T21:54:49.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 802888704 bytes 2026-02-20T21:54:49.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 811869184 bytes 2026-02-20T21:54:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 830170112 bytes 2026-02-20T21:54:49.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 872632320 bytes 2026-02-20T21:54:49.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 926353408 bytes 2026-02-20T21:54:50.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 957348864 bytes 2026-02-20T21:54:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 974579712 bytes 2026-02-20T21:54:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1023345664 bytes 2026-02-20T21:54:50.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1058408448 bytes 2026-02-20T21:54:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1121486848 bytes 2026-02-20T21:54:50.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1187905536 bytes 2026-02-20T21:54:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1200468992 bytes 2026-02-20T21:54:50.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1285314560 bytes 2026-02-20T21:54:50.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1359631360 bytes 2026-02-20T21:54:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1420296192 bytes 2026-02-20T21:54:50.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1432015872 bytes 2026-02-20T21:54:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1505451008 bytes 2026-02-20T21:54:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1555396608 bytes 2026-02-20T21:54:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1586098176 bytes 2026-02-20T21:54:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1657396224 bytes 2026-02-20T21:54:51.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1658022912 bytes 2026-02-20T21:54:51.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1722130432 bytes 2026-02-20T21:54:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1759868928 bytes 2026-02-20T21:54:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1760477184 bytes 2026-02-20T21:54:51.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1802972160 bytes 2026-02-20T21:54:51.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1857529856 bytes 2026-02-20T21:54:51.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1916431360 bytes 2026-02-20T21:54:51.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2015078400 bytes 2026-02-20T21:54:51.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2087535616 bytes 2026-02-20T21:54:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2123194368 bytes 2026-02-20T21:54:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2123787264 bytes 2026-02-20T21:54:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2154525696 bytes 2026-02-20T21:54:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2168508416 bytes 2026-02-20T21:54:51.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2181783552 bytes 2026-02-20T21:54:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2185887744 bytes 2026-02-20T21:54:51.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2222786560 bytes 2026-02-20T21:54:52.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2281435136 bytes 2026-02-20T21:54:52.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2283723776 bytes 2026-02-20T21:54:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2341359616 bytes 2026-02-20T21:54:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2347930624 bytes 2026-02-20T21:54:52.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2406967296 bytes 2026-02-20T21:54:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2494070784 bytes 2026-02-20T21:54:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2584815616 bytes 2026-02-20T21:54:52.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2698117120 bytes 2026-02-20T21:54:52.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2818825216 bytes 2026-02-20T21:54:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2920771584 bytes 2026-02-20T21:54:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2933106688 bytes 2026-02-20T21:54:52.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2947461120 bytes 2026-02-20T21:54:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2993683456 bytes 2026-02-20T21:54:52.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3127939072 bytes 2026-02-20T21:54:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3153753088 bytes 2026-02-20T21:54:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3247232000 bytes 2026-02-20T21:54:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3310432256 bytes 2026-02-20T21:54:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3365681152 bytes 2026-02-20T21:54:53.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3366300672 bytes 2026-02-20T21:54:53.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3439256576 bytes 2026-02-20T21:54:53.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3522441216 bytes 2026-02-20T21:54:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3566755840 bytes 2026-02-20T21:54:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3588243456 bytes 2026-02-20T21:54:53.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3621197824 bytes 2026-02-20T21:54:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3657831424 bytes 2026-02-20T21:54:53.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3720847360 bytes 2026-02-20T21:54:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3764675584 bytes 2026-02-20T21:54:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3775824896 bytes 2026-02-20T21:54:53.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3831368704 bytes 2026-02-20T21:54:53.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3883927552 bytes 2026-02-20T21:54:53.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3932608512 bytes 2026-02-20T21:54:53.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3960877056 bytes 2026-02-20T21:54:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3983069184 bytes 2026-02-20T21:54:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3996744704 bytes 2026-02-20T21:54:53.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4049085440 bytes 2026-02-20T21:54:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4110358528 bytes 2026-02-20T21:54:53.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4154271744 bytes 2026-02-20T21:54:53.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4154861568 bytes 2026-02-20T21:54:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4196189184 bytes 2026-02-20T21:54:54.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4206943232 bytes 2026-02-20T21:54:54.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4273874944 bytes 2026-02-20T21:54:54.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4307562496 bytes 2026-02-20T21:54:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4308165632 bytes 2026-02-20T21:54:54.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4325113856 bytes 2026-02-20T21:54:54.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4412769280 bytes 2026-02-20T21:54:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4489958400 bytes 2026-02-20T21:54:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4501937152 bytes 2026-02-20T21:54:54.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4556804096 bytes 2026-02-20T21:54:54.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4623199232 bytes 2026-02-20T21:54:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4667657216 bytes 2026-02-20T21:54:54.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4692314112 bytes 2026-02-20T21:54:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4734483456 bytes 2026-02-20T21:54:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4744092672 bytes 2026-02-20T21:54:54.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4755495936 bytes 2026-02-20T21:54:54.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4806304768 bytes 2026-02-20T21:54:54.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4872971264 bytes 2026-02-20T21:54:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4908532736 bytes 2026-02-20T21:54:54.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4919702528 bytes 2026-02-20T21:54:54.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4941940736 bytes 2026-02-20T21:54:54.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4969108480 bytes 2026-02-20T21:54:55.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5046292480 bytes 2026-02-20T21:54:55.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5104196608 bytes 2026-02-20T21:54:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5162555392 bytes 2026-02-20T21:54:55.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5193798656 bytes 2026-02-20T21:54:55.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5221291008 bytes 2026-02-20T21:54:55.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5239065600 bytes 2026-02-20T21:54:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5251876864 bytes 2026-02-20T21:54:55.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5298214912 bytes 2026-02-20T21:54:55.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5326749696 bytes 2026-02-20T21:54:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5335568384 bytes 2026-02-20T21:54:55.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5360485376 bytes 2026-02-20T21:54:55.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5430036480 bytes 2026-02-20T21:54:55.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5458296832 bytes 2026-02-20T21:54:55.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5477237760 bytes 2026-02-20T21:54:55.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5482292224 bytes 2026-02-20T21:54:55.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5534078976 bytes 2026-02-20T21:54:55.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:54:55.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:54:55.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_43 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 snapshot 128M 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_43 128M 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:54:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:54:55.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:54:55.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_43 2026-02-20T21:54:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:54:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:54:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:54:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_43 snapshot 2026-02-20T21:54:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T21:54:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:54:55.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_43 snapshot 2026-02-20T21:54:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:54:56.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_43 2026-02-20T21:54:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_43': 2026-02-20T21:54:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:54:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:54:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:54:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 148b7626b179 2026-02-20T21:54:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.148b7626b179 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:55 2026 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:55 2026 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:55 2026 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: b3411393-1507-494e-b3af-7d1d3d485652 2026-02-20T21:54:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:54:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:54:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_43 2026-02-20T21:54:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:54:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:54:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T21:54:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:54:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=10 2026-02-20T21:54:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:54:56.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:54:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 97798144 bytes 2026-02-20T21:54:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 107977728 bytes 2026-02-20T21:54:56.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 154456064 bytes 2026-02-20T21:54:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 238446592 bytes 2026-02-20T21:54:57.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 260317184 bytes 2026-02-20T21:54:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 272096256 bytes 2026-02-20T21:54:57.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 283827200 bytes 2026-02-20T21:54:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 314250240 bytes 2026-02-20T21:54:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 343668736 bytes 2026-02-20T21:54:57.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 373449728 bytes 2026-02-20T21:54:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 384453632 bytes 2026-02-20T21:54:57.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 402946048 bytes 2026-02-20T21:54:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 418371584 bytes 2026-02-20T21:54:57.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 486112256 bytes 2026-02-20T21:54:57.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 533403648 bytes 2026-02-20T21:54:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 572110848 bytes 2026-02-20T21:54:57.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 630650880 bytes 2026-02-20T21:54:57.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 668464128 bytes 2026-02-20T21:54:57.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 710602752 bytes 2026-02-20T21:54:57.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 768634880 bytes 2026-02-20T21:54:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 819309568 bytes 2026-02-20T21:54:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 856591360 bytes 2026-02-20T21:54:57.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 896345088 bytes 2026-02-20T21:54:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 960519168 bytes 2026-02-20T21:54:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1056109568 bytes 2026-02-20T21:54:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1091396608 bytes 2026-02-20T21:54:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1114072064 bytes 2026-02-20T21:54:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1205470208 bytes 2026-02-20T21:54:58.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1267766272 bytes 2026-02-20T21:54:58.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1307091968 bytes 2026-02-20T21:54:58.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1307694080 bytes 2026-02-20T21:54:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1380135936 bytes 2026-02-20T21:54:58.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1473717248 bytes 2026-02-20T21:54:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1492995072 bytes 2026-02-20T21:54:58.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1523012608 bytes 2026-02-20T21:54:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1573986304 bytes 2026-02-20T21:54:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1597893632 bytes 2026-02-20T21:54:58.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1671367680 bytes 2026-02-20T21:54:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1711458304 bytes 2026-02-20T21:54:58.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1766043648 bytes 2026-02-20T21:54:58.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1837101056 bytes 2026-02-20T21:54:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1924066304 bytes 2026-02-20T21:54:59.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1946033152 bytes 2026-02-20T21:54:59.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2002116608 bytes 2026-02-20T21:54:59.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2047652864 bytes 2026-02-20T21:54:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2081026048 bytes 2026-02-20T21:54:59.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2095867904 bytes 2026-02-20T21:54:59.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2169811968 bytes 2026-02-20T21:54:59.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2180389888 bytes 2026-02-20T21:54:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2189787136 bytes 2026-02-20T21:54:59.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2223269888 bytes 2026-02-20T21:54:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2271869952 bytes 2026-02-20T21:54:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2329750528 bytes 2026-02-20T21:54:59.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2359258112 bytes 2026-02-20T21:54:59.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2435550208 bytes 2026-02-20T21:54:59.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2450132992 bytes 2026-02-20T21:54:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2489737216 bytes 2026-02-20T21:54:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2514419712 bytes 2026-02-20T21:54:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2515035136 bytes 2026-02-20T21:54:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2602484736 bytes 2026-02-20T21:54:59.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2608607232 bytes 2026-02-20T21:54:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2633484288 bytes 2026-02-20T21:54:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2646751232 bytes 2026-02-20T21:55:00.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2679975936 bytes 2026-02-20T21:55:00.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2687878144 bytes 2026-02-20T21:55:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2745934848 bytes 2026-02-20T21:55:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2799417344 bytes 2026-02-20T21:55:00.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2811661312 bytes 2026-02-20T21:55:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2868295680 bytes 2026-02-20T21:55:00.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2881474560 bytes 2026-02-20T21:55:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2920073216 bytes 2026-02-20T21:55:00.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3024254976 bytes 2026-02-20T21:55:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3038478336 bytes 2026-02-20T21:55:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3077764096 bytes 2026-02-20T21:55:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3117959168 bytes 2026-02-20T21:55:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3160044544 bytes 2026-02-20T21:55:00.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3213420544 bytes 2026-02-20T21:55:00.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3273519104 bytes 2026-02-20T21:55:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3309107200 bytes 2026-02-20T21:55:00.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3310511104 bytes 2026-02-20T21:55:00.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3367430144 bytes 2026-02-20T21:55:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3386205184 bytes 2026-02-20T21:55:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3409523712 bytes 2026-02-20T21:55:00.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3470856192 bytes 2026-02-20T21:55:01.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3540844544 bytes 2026-02-20T21:55:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3571469312 bytes 2026-02-20T21:55:01.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3584313344 bytes 2026-02-20T21:55:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3644286976 bytes 2026-02-20T21:55:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3698505728 bytes 2026-02-20T21:55:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3768344576 bytes 2026-02-20T21:55:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3826907136 bytes 2026-02-20T21:55:01.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3882750976 bytes 2026-02-20T21:55:01.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3928788992 bytes 2026-02-20T21:55:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4044490752 bytes 2026-02-20T21:55:01.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4090856448 bytes 2026-02-20T21:55:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4109111296 bytes 2026-02-20T21:55:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4186789888 bytes 2026-02-20T21:55:01.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4239599616 bytes 2026-02-20T21:55:01.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4264941568 bytes 2026-02-20T21:55:01.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4265550848 bytes 2026-02-20T21:55:01.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4306941952 bytes 2026-02-20T21:55:01.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4307543040 bytes 2026-02-20T21:55:01.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4350336000 bytes 2026-02-20T21:55:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4383195136 bytes 2026-02-20T21:55:02.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4394131456 bytes 2026-02-20T21:55:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4429284352 bytes 2026-02-20T21:55:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4513441792 bytes 2026-02-20T21:55:02.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4547458048 bytes 2026-02-20T21:55:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4648978432 bytes 2026-02-20T21:55:02.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4681797632 bytes 2026-02-20T21:55:02.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4682451968 bytes 2026-02-20T21:55:02.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4699189248 bytes 2026-02-20T21:55:02.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4736058368 bytes 2026-02-20T21:55:02.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4802762752 bytes 2026-02-20T21:55:02.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4817192960 bytes 2026-02-20T21:55:02.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4892331008 bytes 2026-02-20T21:55:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4929569792 bytes 2026-02-20T21:55:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4954471424 bytes 2026-02-20T21:55:02.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4989745152 bytes 2026-02-20T21:55:02.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5052968960 bytes 2026-02-20T21:55:02.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5053588480 bytes 2026-02-20T21:55:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5157084160 bytes 2026-02-20T21:55:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5223834624 bytes 2026-02-20T21:55:02.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5243453440 bytes 2026-02-20T21:55:03.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5258701824 bytes 2026-02-20T21:55:03.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5279365120 bytes 2026-02-20T21:55:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5279924224 bytes 2026-02-20T21:55:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5292190720 bytes 2026-02-20T21:55:03.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5310607360 bytes 2026-02-20T21:55:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5425165312 bytes 2026-02-20T21:55:03.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5467214848 bytes 2026-02-20T21:55:03.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5508306944 bytes 2026-02-20T21:55:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5569028096 bytes 2026-02-20T21:55:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5569633280 bytes 2026-02-20T21:55:03.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5684014080 bytes 2026-02-20T21:55:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5702054912 bytes 2026-02-20T21:55:03.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5741767680 bytes 2026-02-20T21:55:03.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5757378560 bytes 2026-02-20T21:55:03.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5804963840 bytes 2026-02-20T21:55:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5867702272 bytes 2026-02-20T21:55:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5909168128 bytes 2026-02-20T21:55:03.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5942300672 bytes 2026-02-20T21:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5988294656 bytes 2026-02-20T21:55:03.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5988956160 bytes 2026-02-20T21:55:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6016581632 bytes 2026-02-20T21:55:04.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6063173632 bytes 2026-02-20T21:55:04.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6109149184 bytes 2026-02-20T21:55:04.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6132398080 bytes 2026-02-20T21:55:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6180842496 bytes 2026-02-20T21:55:04.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6262892544 bytes 2026-02-20T21:55:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6276060160 bytes 2026-02-20T21:55:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6291919872 bytes 2026-02-20T21:55:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6344372224 bytes 2026-02-20T21:55:04.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6382205952 bytes 2026-02-20T21:55:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6457321472 bytes 2026-02-20T21:55:04.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6473063424 bytes 2026-02-20T21:55:04.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6473665536 bytes 2026-02-20T21:55:04.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6504375296 bytes 2026-02-20T21:55:04.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6566759424 bytes 2026-02-20T21:55:04.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6622519296 bytes 2026-02-20T21:55:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6696887296 bytes 2026-02-20T21:55:04.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6718727168 bytes 2026-02-20T21:55:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6768262144 bytes 2026-02-20T21:55:04.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6821385216 bytes 2026-02-20T21:55:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6844943360 bytes 2026-02-20T21:55:05.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6899681280 bytes 2026-02-20T21:55:05.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6919739392 bytes 2026-02-20T21:55:05.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6958400512 bytes 2026-02-20T21:55:05.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6998019072 bytes 2026-02-20T21:55:05.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7153536000 bytes 2026-02-20T21:55:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7188560896 bytes 2026-02-20T21:55:05.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7218095104 bytes 2026-02-20T21:55:05.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7289079808 bytes 2026-02-20T21:55:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7293562880 bytes 2026-02-20T21:55:05.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7326837760 bytes 2026-02-20T21:55:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7368522752 bytes 2026-02-20T21:55:05.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7399844864 bytes 2026-02-20T21:55:05.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7409688576 bytes 2026-02-20T21:55:05.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7416467456 bytes 2026-02-20T21:55:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7425646592 bytes 2026-02-20T21:55:05.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7440878592 bytes 2026-02-20T21:55:05.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7512029184 bytes 2026-02-20T21:55:05.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7547504640 bytes 2026-02-20T21:55:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7644861440 bytes 2026-02-20T21:55:05.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7703522304 bytes 2026-02-20T21:55:06.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7772505088 bytes 2026-02-20T21:55:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7855595520 bytes 2026-02-20T21:55:06.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7910674432 bytes 2026-02-20T21:55:06.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7911640064 bytes 2026-02-20T21:55:06.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7990408192 bytes 2026-02-20T21:55:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8075780096 bytes 2026-02-20T21:55:06.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8118403072 bytes 2026-02-20T21:55:06.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8120371200 bytes 2026-02-20T21:55:06.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8150376448 bytes 2026-02-20T21:55:06.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8171930624 bytes 2026-02-20T21:55:06.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8187727872 bytes 2026-02-20T21:55:06.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8215408640 bytes 2026-02-20T21:55:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8241226752 bytes 2026-02-20T21:55:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8313009152 bytes 2026-02-20T21:55:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8374979584 bytes 2026-02-20T21:55:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:55:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:55:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:55:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_44 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 snapshot 128M 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:55:06.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_44 128M 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:06.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_44 2026-02-20T21:55:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:55:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:55:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_44 snapshot 2026-02-20T21:55:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T21:55:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_44 snapshot 2026-02-20T21:55:07.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:55:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_44 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_44': 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 1497128642b8 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.1497128642b8 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:06 2026 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:06 2026 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:06 2026 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:55:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 9c7289c8-3641-48c4-aeec-d1a3caf6cc21 2026-02-20T21:55:07.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:55:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:55:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_44 2026-02-20T21:55:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T21:55:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:55:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T21:55:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:55:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:55:07.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 31514624 bytes 2026-02-20T21:55:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 96157696 bytes 2026-02-20T21:55:08.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 145403904 bytes 2026-02-20T21:55:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 173360128 bytes 2026-02-20T21:55:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 212161536 bytes 2026-02-20T21:55:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 248850432 bytes 2026-02-20T21:55:08.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 313549824 bytes 2026-02-20T21:55:08.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 324888576 bytes 2026-02-20T21:55:08.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 354937856 bytes 2026-02-20T21:55:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 396487680 bytes 2026-02-20T21:55:08.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 442246144 bytes 2026-02-20T21:55:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 514025472 bytes 2026-02-20T21:55:08.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 569706496 bytes 2026-02-20T21:55:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 641370112 bytes 2026-02-20T21:55:08.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 674642944 bytes 2026-02-20T21:55:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 722174976 bytes 2026-02-20T21:55:08.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 774813696 bytes 2026-02-20T21:55:08.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 843361280 bytes 2026-02-20T21:55:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 908183552 bytes 2026-02-20T21:55:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 955411456 bytes 2026-02-20T21:55:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 979736576 bytes 2026-02-20T21:55:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1088098304 bytes 2026-02-20T21:55:09.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1111161856 bytes 2026-02-20T21:55:09.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1123738624 bytes 2026-02-20T21:55:09.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1183801344 bytes 2026-02-20T21:55:09.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1229139968 bytes 2026-02-20T21:55:09.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1271772160 bytes 2026-02-20T21:55:09.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1294365696 bytes 2026-02-20T21:55:09.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1338757120 bytes 2026-02-20T21:55:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1378727936 bytes 2026-02-20T21:55:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1393835008 bytes 2026-02-20T21:55:09.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1399480320 bytes 2026-02-20T21:55:09.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1417244672 bytes 2026-02-20T21:55:09.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1468012544 bytes 2026-02-20T21:55:09.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1520153600 bytes 2026-02-20T21:55:09.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1553957888 bytes 2026-02-20T21:55:09.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1614753792 bytes 2026-02-20T21:55:09.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1687988224 bytes 2026-02-20T21:55:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1727301632 bytes 2026-02-20T21:55:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1773149184 bytes 2026-02-20T21:55:10.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1809225728 bytes 2026-02-20T21:55:10.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1859482624 bytes 2026-02-20T21:55:10.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1942697984 bytes 2026-02-20T21:55:10.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2010188800 bytes 2026-02-20T21:55:10.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2115244032 bytes 2026-02-20T21:55:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2216584192 bytes 2026-02-20T21:55:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2238767104 bytes 2026-02-20T21:55:10.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2263018496 bytes 2026-02-20T21:55:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2356819968 bytes 2026-02-20T21:55:10.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2407497728 bytes 2026-02-20T21:55:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2441969664 bytes 2026-02-20T21:55:10.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2478656512 bytes 2026-02-20T21:55:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2578380800 bytes 2026-02-20T21:55:10.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2665667584 bytes 2026-02-20T21:55:10.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2756588544 bytes 2026-02-20T21:55:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2802254848 bytes 2026-02-20T21:55:11.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2820566016 bytes 2026-02-20T21:55:11.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2838455296 bytes 2026-02-20T21:55:11.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2922545152 bytes 2026-02-20T21:55:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2947041280 bytes 2026-02-20T21:55:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2968075264 bytes 2026-02-20T21:55:11.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2985779200 bytes 2026-02-20T21:55:11.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 3003368448 bytes 2026-02-20T21:55:11.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3046217728 bytes 2026-02-20T21:55:11.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3060564992 bytes 2026-02-20T21:55:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3085767680 bytes 2026-02-20T21:55:11.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3132162048 bytes 2026-02-20T21:55:11.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3173341184 bytes 2026-02-20T21:55:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3232401408 bytes 2026-02-20T21:55:11.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3241373696 bytes 2026-02-20T21:55:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3268930560 bytes 2026-02-20T21:55:11.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3287813120 bytes 2026-02-20T21:55:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3359791104 bytes 2026-02-20T21:55:11.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3391312896 bytes 2026-02-20T21:55:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3400029184 bytes 2026-02-20T21:55:11.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3465601024 bytes 2026-02-20T21:55:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3494305792 bytes 2026-02-20T21:55:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3610727424 bytes 2026-02-20T21:55:12.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3651937280 bytes 2026-02-20T21:55:12.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3685471232 bytes 2026-02-20T21:55:12.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3736582144 bytes 2026-02-20T21:55:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3770455040 bytes 2026-02-20T21:55:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3824322560 bytes 2026-02-20T21:55:12.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3883541504 bytes 2026-02-20T21:55:12.383 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-20T21:55:12.384 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-20T21:55:12.384 INFO:tasks.rbd_mirror_thrash:waiting for 3.178844453677514 secs before reviving daemons 2026-02-20T21:55:12.384 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T21:55:12.379+0000 7f1afef63640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 29846) UID: 1000 2026-02-20T21:55:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3908802560 bytes 2026-02-20T21:55:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3909425152 bytes 2026-02-20T21:55:12.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3937277952 bytes 2026-02-20T21:55:12.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3966872576 bytes 2026-02-20T21:55:12.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4008656896 bytes 2026-02-20T21:55:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4032594944 bytes 2026-02-20T21:55:12.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4078670848 bytes 2026-02-20T21:55:12.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4149294080 bytes 2026-02-20T21:55:12.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4173560832 bytes 2026-02-20T21:55:12.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4174168064 bytes 2026-02-20T21:55:12.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4257867776 bytes 2026-02-20T21:55:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4309747712 bytes 2026-02-20T21:55:12.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4354949120 bytes 2026-02-20T21:55:12.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4361416704 bytes 2026-02-20T21:55:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4390827008 bytes 2026-02-20T21:55:13.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4444513280 bytes 2026-02-20T21:55:13.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4459386880 bytes 2026-02-20T21:55:13.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4514330624 bytes 2026-02-20T21:55:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4527692800 bytes 2026-02-20T21:55:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4573727744 bytes 2026-02-20T21:55:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4574356480 bytes 2026-02-20T21:55:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4603756544 bytes 2026-02-20T21:55:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4677180416 bytes 2026-02-20T21:55:13.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4775989248 bytes 2026-02-20T21:55:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4844884992 bytes 2026-02-20T21:55:13.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4854234112 bytes 2026-02-20T21:55:13.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4854847488 bytes 2026-02-20T21:55:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4898239488 bytes 2026-02-20T21:55:13.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4925546496 bytes 2026-02-20T21:55:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4988356608 bytes 2026-02-20T21:55:13.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5022356480 bytes 2026-02-20T21:55:13.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5072545792 bytes 2026-02-20T21:55:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5174264832 bytes 2026-02-20T21:55:13.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5198046208 bytes 2026-02-20T21:55:14.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5240186880 bytes 2026-02-20T21:55:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5255667712 bytes 2026-02-20T21:55:14.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5300350976 bytes 2026-02-20T21:55:14.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5384107008 bytes 2026-02-20T21:55:14.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5464273920 bytes 2026-02-20T21:55:14.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5534454784 bytes 2026-02-20T21:55:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5580492800 bytes 2026-02-20T21:55:14.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5640168448 bytes 2026-02-20T21:55:14.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5697062912 bytes 2026-02-20T21:55:14.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5705213952 bytes 2026-02-20T21:55:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5725312000 bytes 2026-02-20T21:55:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5814764544 bytes 2026-02-20T21:55:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5842498560 bytes 2026-02-20T21:55:14.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5899065344 bytes 2026-02-20T21:55:14.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5922478080 bytes 2026-02-20T21:55:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5974898688 bytes 2026-02-20T21:55:14.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5988128768 bytes 2026-02-20T21:55:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 6031190016 bytes 2026-02-20T21:55:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 6063703040 bytes 2026-02-20T21:55:14.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6087933952 bytes 2026-02-20T21:55:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6169920512 bytes 2026-02-20T21:55:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6221417472 bytes 2026-02-20T21:55:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6311123968 bytes 2026-02-20T21:55:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6358910976 bytes 2026-02-20T21:55:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6435917824 bytes 2026-02-20T21:55:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6477364224 bytes 2026-02-20T21:55:15.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6554294272 bytes 2026-02-20T21:55:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6579632128 bytes 2026-02-20T21:55:15.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6582705152 bytes 2026-02-20T21:55:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6640546816 bytes 2026-02-20T21:55:15.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6674854912 bytes 2026-02-20T21:55:15.563 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-20T21:55:15.564 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:55:15.564 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-20T21:55:15.564 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T21:55:15.564 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T21:55:15.567 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T21:55:15.567 INFO:tasks.rbd_mirror_thrash:waiting for 72.98870478052547 secs before thrashing 2026-02-20T21:55:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6711153664 bytes 2026-02-20T21:55:15.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6780261376 bytes 2026-02-20T21:55:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6786398208 bytes 2026-02-20T21:55:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6833191936 bytes 2026-02-20T21:55:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6905782272 bytes 2026-02-20T21:55:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6909118464 bytes 2026-02-20T21:55:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6947111936 bytes 2026-02-20T21:55:15.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 7060300800 bytes 2026-02-20T21:55:15.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 7105094656 bytes 2026-02-20T21:55:16.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 7119067136 bytes 2026-02-20T21:55:16.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 7143488512 bytes 2026-02-20T21:55:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7145904128 bytes 2026-02-20T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7161793536 bytes 2026-02-20T21:55:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7207614464 bytes 2026-02-20T21:55:16.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7237703680 bytes 2026-02-20T21:55:16.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7258353664 bytes 2026-02-20T21:55:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7268557824 bytes 2026-02-20T21:55:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7314934784 bytes 2026-02-20T21:55:16.420 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3842687496949705) > weight (0.25) 2026-02-20T21:55:16.420 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.655570181554304) > weight (0.25) 2026-02-20T21:55:16.420 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-20T21:55:16.421 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-20T21:55:16.421 INFO:tasks.rbd_mirror_thrash:waiting for 7.541624952612125 secs before reviving daemons 2026-02-20T21:55:16.422 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T21:55:16.419+0000 7f794ea65640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 30256) UID: 1000 2026-02-20T21:55:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7421328384 bytes 2026-02-20T21:55:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7467747328 bytes 2026-02-20T21:55:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7514445824 bytes 2026-02-20T21:55:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7531206656 bytes 2026-02-20T21:55:16.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7558862848 bytes 2026-02-20T21:55:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7638733824 bytes 2026-02-20T21:55:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7719109632 bytes 2026-02-20T21:55:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:55:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:55:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:55:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_45 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 snapshot 128M 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:55:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_45 128M 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_45 2026-02-20T21:55:16.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:55:16.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:16.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:55:16.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:16.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_45 snapshot 2026-02-20T21:55:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T21:55:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:16.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_45 snapshot 2026-02-20T21:55:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:55:17.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_45 2026-02-20T21:55:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_45': 2026-02-20T21:55:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:55:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:55:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 14a9e811c227 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.14a9e811c227 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:16 2026 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:16 2026 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:16 2026 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:55:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: cf134cc1-67a2-4edd-85cf-ecb315b92e21 2026-02-20T21:55:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:55:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:55:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_45 2026-02-20T21:55:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T21:55:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:55:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=10 2026-02-20T21:55:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:55:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:55:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 66305024 bytes 2026-02-20T21:55:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 92218368 bytes 2026-02-20T21:55:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 131506176 bytes 2026-02-20T21:55:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 224270336 bytes 2026-02-20T21:55:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 261602304 bytes 2026-02-20T21:55:18.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 262193152 bytes 2026-02-20T21:55:18.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 346756096 bytes 2026-02-20T21:55:18.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 352040960 bytes 2026-02-20T21:55:18.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 396203008 bytes 2026-02-20T21:55:18.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 440042496 bytes 2026-02-20T21:55:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 563547136 bytes 2026-02-20T21:55:18.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 593374208 bytes 2026-02-20T21:55:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 633702400 bytes 2026-02-20T21:55:18.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 693555200 bytes 2026-02-20T21:55:18.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 778385408 bytes 2026-02-20T21:55:18.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 797815808 bytes 2026-02-20T21:55:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 820262912 bytes 2026-02-20T21:55:18.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 844462080 bytes 2026-02-20T21:55:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 892779520 bytes 2026-02-20T21:55:19.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 938204160 bytes 2026-02-20T21:55:19.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 965814272 bytes 2026-02-20T21:55:19.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 974925824 bytes 2026-02-20T21:55:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 990571520 bytes 2026-02-20T21:55:19.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1021967360 bytes 2026-02-20T21:55:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1046666240 bytes 2026-02-20T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1094301696 bytes 2026-02-20T21:55:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1103533056 bytes 2026-02-20T21:55:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1144611840 bytes 2026-02-20T21:55:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1179971584 bytes 2026-02-20T21:55:19.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1214445568 bytes 2026-02-20T21:55:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1289731072 bytes 2026-02-20T21:55:19.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1372473344 bytes 2026-02-20T21:55:19.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1392461824 bytes 2026-02-20T21:55:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1469725696 bytes 2026-02-20T21:55:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1529971712 bytes 2026-02-20T21:55:19.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1538215936 bytes 2026-02-20T21:55:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1555922944 bytes 2026-02-20T21:55:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1567527936 bytes 2026-02-20T21:55:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1592739840 bytes 2026-02-20T21:55:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1621165056 bytes 2026-02-20T21:55:20.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1639000064 bytes 2026-02-20T21:55:20.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1701041152 bytes 2026-02-20T21:55:20.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1730110464 bytes 2026-02-20T21:55:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1796362240 bytes 2026-02-20T21:55:20.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1813380096 bytes 2026-02-20T21:55:20.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1813994496 bytes 2026-02-20T21:55:20.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1881491456 bytes 2026-02-20T21:55:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1920294912 bytes 2026-02-20T21:55:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1950401536 bytes 2026-02-20T21:55:20.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1967761408 bytes 2026-02-20T21:55:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2043941888 bytes 2026-02-20T21:55:20.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2073912320 bytes 2026-02-20T21:55:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2134528000 bytes 2026-02-20T21:55:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2195796992 bytes 2026-02-20T21:55:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2306033664 bytes 2026-02-20T21:55:20.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2327759872 bytes 2026-02-20T21:55:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2403752960 bytes 2026-02-20T21:55:20.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2433684480 bytes 2026-02-20T21:55:20.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2476901376 bytes 2026-02-20T21:55:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2481707008 bytes 2026-02-20T21:55:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2484542464 bytes 2026-02-20T21:55:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2586799104 bytes 2026-02-20T21:55:21.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2672835584 bytes 2026-02-20T21:55:21.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2732638208 bytes 2026-02-20T21:55:21.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2733263872 bytes 2026-02-20T21:55:21.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2756349952 bytes 2026-02-20T21:55:21.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2778705920 bytes 2026-02-20T21:55:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2799710208 bytes 2026-02-20T21:55:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2862798848 bytes 2026-02-20T21:55:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2865684480 bytes 2026-02-20T21:55:21.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2895817728 bytes 2026-02-20T21:55:21.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2896433152 bytes 2026-02-20T21:55:21.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 2915604480 bytes 2026-02-20T21:55:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 2989950976 bytes 2026-02-20T21:55:21.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3016072192 bytes 2026-02-20T21:55:21.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3020810240 bytes 2026-02-20T21:55:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3031053312 bytes 2026-02-20T21:55:21.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3078658048 bytes 2026-02-20T21:55:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3107989504 bytes 2026-02-20T21:55:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3172686848 bytes 2026-02-20T21:55:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3240452096 bytes 2026-02-20T21:55:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3418248192 bytes 2026-02-20T21:55:22.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3523075072 bytes 2026-02-20T21:55:22.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3570239488 bytes 2026-02-20T21:55:22.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3650069504 bytes 2026-02-20T21:55:22.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3670351872 bytes 2026-02-20T21:55:22.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3697362944 bytes 2026-02-20T21:55:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3729768448 bytes 2026-02-20T21:55:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3799602176 bytes 2026-02-20T21:55:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3812329472 bytes 2026-02-20T21:55:22.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3861489664 bytes 2026-02-20T21:55:22.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3893354496 bytes 2026-02-20T21:55:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3980849152 bytes 2026-02-20T21:55:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4044521472 bytes 2026-02-20T21:55:22.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4095077376 bytes 2026-02-20T21:55:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4141533184 bytes 2026-02-20T21:55:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4145409024 bytes 2026-02-20T21:55:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4166860800 bytes 2026-02-20T21:55:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4212805632 bytes 2026-02-20T21:55:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4236046336 bytes 2026-02-20T21:55:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4262068224 bytes 2026-02-20T21:55:23.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4284732416 bytes 2026-02-20T21:55:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4331999232 bytes 2026-02-20T21:55:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4376593408 bytes 2026-02-20T21:55:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4394597376 bytes 2026-02-20T21:55:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4411420672 bytes 2026-02-20T21:55:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4449078272 bytes 2026-02-20T21:55:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4491412480 bytes 2026-02-20T21:55:23.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4545971200 bytes 2026-02-20T21:55:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4575860736 bytes 2026-02-20T21:55:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4640472064 bytes 2026-02-20T21:55:23.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4666230784 bytes 2026-02-20T21:55:23.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4735242240 bytes 2026-02-20T21:55:23.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4766954496 bytes 2026-02-20T21:55:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4803246080 bytes 2026-02-20T21:55:23.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4862609408 bytes 2026-02-20T21:55:23.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4883842048 bytes 2026-02-20T21:55:23.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4949293056 bytes 2026-02-20T21:55:23.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4975474688 bytes 2026-02-20T21:55:23.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5016673280 bytes 2026-02-20T21:55:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5029462016 bytes 2026-02-20T21:55:23.964 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-20T21:55:23.964 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:55:23.964 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-20T21:55:23.964 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T21:55:23.964 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T21:55:23.966 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T21:55:23.966 INFO:tasks.rbd_mirror_thrash:waiting for 116.06244808199585 secs before thrashing 2026-02-20T21:55:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5058414592 bytes 2026-02-20T21:55:24.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5127037952 bytes 2026-02-20T21:55:24.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5226119168 bytes 2026-02-20T21:55:24.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5259787264 bytes 2026-02-20T21:55:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5322268672 bytes 2026-02-20T21:55:24.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5388693504 bytes 2026-02-20T21:55:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5433162752 bytes 2026-02-20T21:55:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5474598912 bytes 2026-02-20T21:55:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5491124224 bytes 2026-02-20T21:55:24.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5567938560 bytes 2026-02-20T21:55:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5568562176 bytes 2026-02-20T21:55:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5586067456 bytes 2026-02-20T21:55:24.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5657954304 bytes 2026-02-20T21:55:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5709740032 bytes 2026-02-20T21:55:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5756529664 bytes 2026-02-20T21:55:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5796910080 bytes 2026-02-20T21:55:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5840961536 bytes 2026-02-20T21:55:24.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5907960832 bytes 2026-02-20T21:55:24.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5961513984 bytes 2026-02-20T21:55:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5982845952 bytes 2026-02-20T21:55:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6000216064 bytes 2026-02-20T21:55:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6025057280 bytes 2026-02-20T21:55:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6032205824 bytes 2026-02-20T21:55:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6081723392 bytes 2026-02-20T21:55:25.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6165950464 bytes 2026-02-20T21:55:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6217108480 bytes 2026-02-20T21:55:25.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6301406208 bytes 2026-02-20T21:55:25.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6310941696 bytes 2026-02-20T21:55:25.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6322167808 bytes 2026-02-20T21:55:25.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6346221568 bytes 2026-02-20T21:55:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6377811968 bytes 2026-02-20T21:55:25.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6418498560 bytes 2026-02-20T21:55:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6445510656 bytes 2026-02-20T21:55:25.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6541372416 bytes 2026-02-20T21:55:25.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6560507904 bytes 2026-02-20T21:55:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6616289280 bytes 2026-02-20T21:55:25.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6657927168 bytes 2026-02-20T21:55:25.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6722997248 bytes 2026-02-20T21:55:25.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6723605504 bytes 2026-02-20T21:55:25.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6766617600 bytes 2026-02-20T21:55:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6817629184 bytes 2026-02-20T21:55:26.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6824924160 bytes 2026-02-20T21:55:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6903160832 bytes 2026-02-20T21:55:26.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6931675136 bytes 2026-02-20T21:55:26.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7027874816 bytes 2026-02-20T21:55:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7059140608 bytes 2026-02-20T21:55:26.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7090121728 bytes 2026-02-20T21:55:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7104792576 bytes 2026-02-20T21:55:26.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7127290880 bytes 2026-02-20T21:55:26.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7136722944 bytes 2026-02-20T21:55:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7183505408 bytes 2026-02-20T21:55:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7247876096 bytes 2026-02-20T21:55:26.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7283715072 bytes 2026-02-20T21:55:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7370556416 bytes 2026-02-20T21:55:26.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7429152768 bytes 2026-02-20T21:55:26.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7447587840 bytes 2026-02-20T21:55:26.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7488713728 bytes 2026-02-20T21:55:26.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7592911872 bytes 2026-02-20T21:55:26.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7593535488 bytes 2026-02-20T21:55:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7619383296 bytes 2026-02-20T21:55:26.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7668759552 bytes 2026-02-20T21:55:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7700277248 bytes 2026-02-20T21:55:27.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7728962560 bytes 2026-02-20T21:55:27.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7775877120 bytes 2026-02-20T21:55:27.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7850570752 bytes 2026-02-20T21:55:27.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7924069376 bytes 2026-02-20T21:55:27.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7985757184 bytes 2026-02-20T21:55:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8070252544 bytes 2026-02-20T21:55:27.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8119222272 bytes 2026-02-20T21:55:27.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8128837632 bytes 2026-02-20T21:55:27.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8182592512 bytes 2026-02-20T21:55:27.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8183193600 bytes 2026-02-20T21:55:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8206050304 bytes 2026-02-20T21:55:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8262161408 bytes 2026-02-20T21:55:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8324284416 bytes 2026-02-20T21:55:27.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8341753856 bytes 2026-02-20T21:55:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8393990144 bytes 2026-02-20T21:55:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8412522496 bytes 2026-02-20T21:55:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:55:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:55:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:55:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_46 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 snapshot 128M 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_46 128M 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:27.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_46 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_46 snapshot 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_46 snapshot 2026-02-20T21:55:28.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:55:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_46 2026-02-20T21:55:28.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_46': 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 14bb8fcfdd78 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.14bb8fcfdd78 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:55:28.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:27 2026 2026-02-20T21:55:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:27 2026 2026-02-20T21:55:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:27 2026 2026-02-20T21:55:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:55:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:55:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 8e7ce785-f9ba-4268-a043-f8b326632069 2026-02-20T21:55:28.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:55:28.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:55:28.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_46 2026-02-20T21:55:28.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:28.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:28.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T21:55:28.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:55:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=7 2026-02-20T21:55:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:55:28.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:55:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 25150464 bytes 2026-02-20T21:55:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 85366784 bytes 2026-02-20T21:55:28.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 145315840 bytes 2026-02-20T21:55:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 177312768 bytes 2026-02-20T21:55:28.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 195113984 bytes 2026-02-20T21:55:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 281873408 bytes 2026-02-20T21:55:28.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 358560768 bytes 2026-02-20T21:55:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 400161792 bytes 2026-02-20T21:55:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 400774144 bytes 2026-02-20T21:55:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 436052992 bytes 2026-02-20T21:55:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 472990720 bytes 2026-02-20T21:55:29.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 547676160 bytes 2026-02-20T21:55:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 572862464 bytes 2026-02-20T21:55:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 636684288 bytes 2026-02-20T21:55:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 657294336 bytes 2026-02-20T21:55:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 706035712 bytes 2026-02-20T21:55:29.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 799385600 bytes 2026-02-20T21:55:29.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 847393792 bytes 2026-02-20T21:55:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 848514048 bytes 2026-02-20T21:55:29.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 941053952 bytes 2026-02-20T21:55:29.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 990157824 bytes 2026-02-20T21:55:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 993157120 bytes 2026-02-20T21:55:29.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1079946240 bytes 2026-02-20T21:55:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1098596352 bytes 2026-02-20T21:55:29.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1113357312 bytes 2026-02-20T21:55:29.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1176479744 bytes 2026-02-20T21:55:29.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1233164288 bytes 2026-02-20T21:55:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1233768448 bytes 2026-02-20T21:55:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1348756480 bytes 2026-02-20T21:55:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1404176384 bytes 2026-02-20T21:55:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1493229568 bytes 2026-02-20T21:55:30.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1513243648 bytes 2026-02-20T21:55:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1577134080 bytes 2026-02-20T21:55:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1608851456 bytes 2026-02-20T21:55:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1641842688 bytes 2026-02-20T21:55:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1718663168 bytes 2026-02-20T21:55:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1735950336 bytes 2026-02-20T21:55:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1759873024 bytes 2026-02-20T21:55:30.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1815130112 bytes 2026-02-20T21:55:30.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1816726528 bytes 2026-02-20T21:55:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1872391168 bytes 2026-02-20T21:55:30.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1932086272 bytes 2026-02-20T21:55:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1975633920 bytes 2026-02-20T21:55:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2006755328 bytes 2026-02-20T21:55:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2044615680 bytes 2026-02-20T21:55:30.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2050594816 bytes 2026-02-20T21:55:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2077511680 bytes 2026-02-20T21:55:30.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2105600000 bytes 2026-02-20T21:55:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2144343040 bytes 2026-02-20T21:55:31.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2277102592 bytes 2026-02-20T21:55:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2341209088 bytes 2026-02-20T21:55:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2398763008 bytes 2026-02-20T21:55:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2438052864 bytes 2026-02-20T21:55:31.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2478160896 bytes 2026-02-20T21:55:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2488230912 bytes 2026-02-20T21:55:31.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2535140352 bytes 2026-02-20T21:55:31.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2576979968 bytes 2026-02-20T21:55:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2633734144 bytes 2026-02-20T21:55:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2700994560 bytes 2026-02-20T21:55:31.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2734022656 bytes 2026-02-20T21:55:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2800276480 bytes 2026-02-20T21:55:31.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2839577600 bytes 2026-02-20T21:55:31.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2880588800 bytes 2026-02-20T21:55:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2950540288 bytes 2026-02-20T21:55:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2963450880 bytes 2026-02-20T21:55:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3007655936 bytes 2026-02-20T21:55:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3084341248 bytes 2026-02-20T21:55:32.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3175564288 bytes 2026-02-20T21:55:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3273407488 bytes 2026-02-20T21:55:32.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3322206208 bytes 2026-02-20T21:55:32.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3379113984 bytes 2026-02-20T21:55:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3397372928 bytes 2026-02-20T21:55:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3452018688 bytes 2026-02-20T21:55:32.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3495017472 bytes 2026-02-20T21:55:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3510951936 bytes 2026-02-20T21:55:32.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3612755968 bytes 2026-02-20T21:55:32.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3632147456 bytes 2026-02-20T21:55:32.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3660719104 bytes 2026-02-20T21:55:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3765444608 bytes 2026-02-20T21:55:32.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3817486336 bytes 2026-02-20T21:55:32.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3862029312 bytes 2026-02-20T21:55:32.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3901647872 bytes 2026-02-20T21:55:32.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3949558784 bytes 2026-02-20T21:55:32.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3975688192 bytes 2026-02-20T21:55:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 4054434816 bytes 2026-02-20T21:55:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 4095287296 bytes 2026-02-20T21:55:33.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 4140701696 bytes 2026-02-20T21:55:33.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4204045312 bytes 2026-02-20T21:55:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4229977088 bytes 2026-02-20T21:55:33.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4250762240 bytes 2026-02-20T21:55:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4259347456 bytes 2026-02-20T21:55:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4287815680 bytes 2026-02-20T21:55:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4391206912 bytes 2026-02-20T21:55:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4444995584 bytes 2026-02-20T21:55:33.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4491173888 bytes 2026-02-20T21:55:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4510891008 bytes 2026-02-20T21:55:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4584157184 bytes 2026-02-20T21:55:33.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4607833088 bytes 2026-02-20T21:55:33.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4631942144 bytes 2026-02-20T21:55:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4679661568 bytes 2026-02-20T21:55:33.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4722106368 bytes 2026-02-20T21:55:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4774632448 bytes 2026-02-20T21:55:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4795785216 bytes 2026-02-20T21:55:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4820568064 bytes 2026-02-20T21:55:33.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4866561024 bytes 2026-02-20T21:55:34.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4927307776 bytes 2026-02-20T21:55:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4980286464 bytes 2026-02-20T21:55:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4995637248 bytes 2026-02-20T21:55:34.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 5003090944 bytes 2026-02-20T21:55:34.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5087040512 bytes 2026-02-20T21:55:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5110273024 bytes 2026-02-20T21:55:34.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5140757504 bytes 2026-02-20T21:55:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5169396736 bytes 2026-02-20T21:55:34.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5218574336 bytes 2026-02-20T21:55:34.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5242978304 bytes 2026-02-20T21:55:34.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5327032320 bytes 2026-02-20T21:55:34.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5370005504 bytes 2026-02-20T21:55:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5370614784 bytes 2026-02-20T21:55:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5385976832 bytes 2026-02-20T21:55:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5422725120 bytes 2026-02-20T21:55:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5477078016 bytes 2026-02-20T21:55:34.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5541442560 bytes 2026-02-20T21:55:34.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5580320768 bytes 2026-02-20T21:55:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5676553216 bytes 2026-02-20T21:55:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5724114944 bytes 2026-02-20T21:55:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5781012480 bytes 2026-02-20T21:55:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5865142272 bytes 2026-02-20T21:55:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5905474560 bytes 2026-02-20T21:55:35.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5979248640 bytes 2026-02-20T21:55:35.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 6008414208 bytes 2026-02-20T21:55:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:55:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:55:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:55:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:35.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_47 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 snapshot 128M 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:35.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_47 128M 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:35.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_47 2026-02-20T21:55:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:55:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:55:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_47 snapshot 2026-02-20T21:55:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T21:55:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_47 snapshot 2026-02-20T21:55:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:55:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_47 2026-02-20T21:55:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_47': 2026-02-20T21:55:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:55:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:55:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:55:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 14c59750ac43 2026-02-20T21:55:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.14c59750ac43 2026-02-20T21:55:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:55:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:55:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:55:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:55:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:35 2026 2026-02-20T21:55:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:35 2026 2026-02-20T21:55:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:35 2026 2026-02-20T21:55:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:55:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:55:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 740d48a2-b3be-43ea-a507-e1c8ff34cb29 2026-02-20T21:55:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:55:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:55:36.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_47 2026-02-20T21:55:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T21:55:36.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:55:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=10 2026-02-20T21:55:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:55:36.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:55:36.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 17291264 bytes 2026-02-20T21:55:36.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 27834368 bytes 2026-02-20T21:55:36.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 30581760 bytes 2026-02-20T21:55:36.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 36492288 bytes 2026-02-20T21:55:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 87876608 bytes 2026-02-20T21:55:36.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 215992320 bytes 2026-02-20T21:55:36.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 273564672 bytes 2026-02-20T21:55:37.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 346842112 bytes 2026-02-20T21:55:37.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 405831680 bytes 2026-02-20T21:55:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 429540352 bytes 2026-02-20T21:55:37.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 430158848 bytes 2026-02-20T21:55:37.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 463920128 bytes 2026-02-20T21:55:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 519841792 bytes 2026-02-20T21:55:37.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 562639872 bytes 2026-02-20T21:55:37.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 565035008 bytes 2026-02-20T21:55:37.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 573671424 bytes 2026-02-20T21:55:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 636279808 bytes 2026-02-20T21:55:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 674232320 bytes 2026-02-20T21:55:37.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 742863872 bytes 2026-02-20T21:55:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 778675200 bytes 2026-02-20T21:55:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 837228544 bytes 2026-02-20T21:55:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 852758528 bytes 2026-02-20T21:55:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 911064064 bytes 2026-02-20T21:55:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 958481408 bytes 2026-02-20T21:55:37.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1005048832 bytes 2026-02-20T21:55:37.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1005668352 bytes 2026-02-20T21:55:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1054032896 bytes 2026-02-20T21:55:37.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1075301376 bytes 2026-02-20T21:55:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1102893056 bytes 2026-02-20T21:55:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1163542528 bytes 2026-02-20T21:55:38.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1182603264 bytes 2026-02-20T21:55:38.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1196422144 bytes 2026-02-20T21:55:38.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1225652224 bytes 2026-02-20T21:55:38.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1271794688 bytes 2026-02-20T21:55:38.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1298436096 bytes 2026-02-20T21:55:38.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1348430848 bytes 2026-02-20T21:55:38.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1349026816 bytes 2026-02-20T21:55:38.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1373363200 bytes 2026-02-20T21:55:38.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1423534080 bytes 2026-02-20T21:55:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1426519040 bytes 2026-02-20T21:55:38.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1484624896 bytes 2026-02-20T21:55:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1539276800 bytes 2026-02-20T21:55:38.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1648778240 bytes 2026-02-20T21:55:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1680695296 bytes 2026-02-20T21:55:38.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1760596992 bytes 2026-02-20T21:55:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1820969984 bytes 2026-02-20T21:55:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1856459776 bytes 2026-02-20T21:55:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1898487808 bytes 2026-02-20T21:55:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1999531008 bytes 2026-02-20T21:55:39.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2047606784 bytes 2026-02-20T21:55:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2113556480 bytes 2026-02-20T21:55:39.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2166114304 bytes 2026-02-20T21:55:39.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2204783616 bytes 2026-02-20T21:55:39.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2245373952 bytes 2026-02-20T21:55:39.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2295170048 bytes 2026-02-20T21:55:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2306813952 bytes 2026-02-20T21:55:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2373449728 bytes 2026-02-20T21:55:39.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2408750080 bytes 2026-02-20T21:55:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2478044160 bytes 2026-02-20T21:55:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2514145280 bytes 2026-02-20T21:55:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2576510976 bytes 2026-02-20T21:55:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2598171648 bytes 2026-02-20T21:55:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2616470528 bytes 2026-02-20T21:55:39.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2632081408 bytes 2026-02-20T21:55:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2697555968 bytes 2026-02-20T21:55:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2764473344 bytes 2026-02-20T21:55:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2799532032 bytes 2026-02-20T21:55:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2821187584 bytes 2026-02-20T21:55:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2859075584 bytes 2026-02-20T21:55:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2943987712 bytes 2026-02-20T21:55:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2981729280 bytes 2026-02-20T21:55:40.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2982320128 bytes 2026-02-20T21:55:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3021282304 bytes 2026-02-20T21:55:40.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3050057728 bytes 2026-02-20T21:55:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3085835264 bytes 2026-02-20T21:55:40.469 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.463+0000 7f254e618640 -1 reset not still connected to 0x56049ef851e0 2026-02-20T21:55:40.469 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.463+0000 7f254e618640 -1 reset not still connected to 0x56049f89a680 2026-02-20T21:55:40.469 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.463+0000 7f254e618640 -1 reset not still connected to 0x56049f89b930 2026-02-20T21:55:40.469 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.463+0000 7fa5e2658640 -1 reset not still connected to 0x55b9033fc750 2026-02-20T21:55:40.470 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.467+0000 7f254e618640 -1 reset not still connected to 0x56049fa49ad0 2026-02-20T21:55:40.470 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.467+0000 7f254e618640 -1 reset not still connected to 0x56049ffa0410 2026-02-20T21:55:40.473 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.467+0000 7fa5e2658640 -1 reset not still connected to 0x55b903980340 2026-02-20T21:55:40.473 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.467+0000 7fa5e2658640 -1 reset not still connected to 0x55b9052d1e10 2026-02-20T21:55:40.474 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b906a13450 2026-02-20T21:55:40.474 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b90790d6c0 2026-02-20T21:55:40.474 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b90790da00 2026-02-20T21:55:40.474 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b91736e9c0 2026-02-20T21:55:40.475 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.471+0000 7f254e618640 -1 reset not still connected to 0x5604a1408340 2026-02-20T21:55:40.475 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.471+0000 7f254e618640 -1 reset not still connected to 0x5604a1408ea0 2026-02-20T21:55:40.475 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.471+0000 7f254e618640 -1 reset not still connected to 0x5604a1409860 2026-02-20T21:55:40.475 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.471+0000 7f254e618640 -1 reset not still connected to 0x5604a1650820 2026-02-20T21:55:40.475 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.471+0000 7f254e618640 -1 reset not still connected to 0x5604acd0d040 2026-02-20T21:55:40.475 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.471+0000 7f254e618640 -1 reset not still connected to 0x5604acd0d1e0 2026-02-20T21:55:40.475 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.471+0000 7f254e618640 -1 reset not still connected to 0x5604be8af450 2026-02-20T21:55:40.475 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b91963a340 2026-02-20T21:55:40.475 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b91c318d00 2026-02-20T21:55:40.476 INFO:tasks.ceph.cluster2.osd.2.trial107.stderr:2026-02-20T21:55:40.471+0000 7f254e618640 -1 reset not still connected to 0x5604c1037e10 2026-02-20T21:55:40.477 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b91ce06a90 2026-02-20T21:55:40.477 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b91eabc270 2026-02-20T21:55:40.477 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b91eabc680 2026-02-20T21:55:40.477 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b9200475f0 2026-02-20T21:55:40.477 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b920047c70 2026-02-20T21:55:40.477 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b92c1084e0 2026-02-20T21:55:40.477 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b92c109380 2026-02-20T21:55:40.477 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b92c1096c0 2026-02-20T21:55:40.477 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:55:40.471+0000 7fa5e2658640 -1 reset not still connected to 0x55b93467c270 2026-02-20T21:55:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3134155776 bytes 2026-02-20T21:55:40.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3165221888 bytes 2026-02-20T21:55:40.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3194579968 bytes 2026-02-20T21:55:40.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3235058688 bytes 2026-02-20T21:55:40.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3272790016 bytes 2026-02-20T21:55:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3305176064 bytes 2026-02-20T21:55:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3320558592 bytes 2026-02-20T21:55:40.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3394140160 bytes 2026-02-20T21:55:40.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3412711424 bytes 2026-02-20T21:55:40.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3413329920 bytes 2026-02-20T21:55:40.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3479241728 bytes 2026-02-20T21:55:41.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3515073536 bytes 2026-02-20T21:55:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3524269056 bytes 2026-02-20T21:55:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3570168832 bytes 2026-02-20T21:55:41.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3654669312 bytes 2026-02-20T21:55:41.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3668889600 bytes 2026-02-20T21:55:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3669522432 bytes 2026-02-20T21:55:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3762627584 bytes 2026-02-20T21:55:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3816960000 bytes 2026-02-20T21:55:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3891964928 bytes 2026-02-20T21:55:41.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3968375808 bytes 2026-02-20T21:55:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 3993616384 bytes 2026-02-20T21:55:41.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4028445696 bytes 2026-02-20T21:55:41.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4059260928 bytes 2026-02-20T21:55:41.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4126365696 bytes 2026-02-20T21:55:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4170480640 bytes 2026-02-20T21:55:41.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4229492736 bytes 2026-02-20T21:55:41.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4282272768 bytes 2026-02-20T21:55:41.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4313058304 bytes 2026-02-20T21:55:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4350015488 bytes 2026-02-20T21:55:42.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4416094208 bytes 2026-02-20T21:55:42.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4446321664 bytes 2026-02-20T21:55:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4481051648 bytes 2026-02-20T21:55:42.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4513108992 bytes 2026-02-20T21:55:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4561021952 bytes 2026-02-20T21:55:42.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4578643968 bytes 2026-02-20T21:55:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4640357376 bytes 2026-02-20T21:55:42.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4672446464 bytes 2026-02-20T21:55:42.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4691017728 bytes 2026-02-20T21:55:42.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4752011264 bytes 2026-02-20T21:55:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4778519552 bytes 2026-02-20T21:55:42.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4789966848 bytes 2026-02-20T21:55:42.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4807011328 bytes 2026-02-20T21:55:42.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4882950144 bytes 2026-02-20T21:55:42.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4961295360 bytes 2026-02-20T21:55:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4973897728 bytes 2026-02-20T21:55:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5016096768 bytes 2026-02-20T21:55:42.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5028703232 bytes 2026-02-20T21:55:42.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5064897536 bytes 2026-02-20T21:55:42.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5113350144 bytes 2026-02-20T21:55:42.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5133372416 bytes 2026-02-20T21:55:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5133996032 bytes 2026-02-20T21:55:43.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5207549952 bytes 2026-02-20T21:55:43.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5208192000 bytes 2026-02-20T21:55:43.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5225643008 bytes 2026-02-20T21:55:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5293751296 bytes 2026-02-20T21:55:43.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5324133376 bytes 2026-02-20T21:55:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5449726976 bytes 2026-02-20T21:55:43.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5566361600 bytes 2026-02-20T21:55:43.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5596291072 bytes 2026-02-20T21:55:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5629517824 bytes 2026-02-20T21:55:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5704183808 bytes 2026-02-20T21:55:43.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5724659712 bytes 2026-02-20T21:55:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5740608512 bytes 2026-02-20T21:55:43.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5766721536 bytes 2026-02-20T21:55:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5852746752 bytes 2026-02-20T21:55:43.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5881882624 bytes 2026-02-20T21:55:43.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5966632960 bytes 2026-02-20T21:55:43.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6003351552 bytes 2026-02-20T21:55:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6057447424 bytes 2026-02-20T21:55:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6122679296 bytes 2026-02-20T21:55:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6138257408 bytes 2026-02-20T21:55:44.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6149291008 bytes 2026-02-20T21:55:44.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6223772672 bytes 2026-02-20T21:55:44.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6253489152 bytes 2026-02-20T21:55:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6254092288 bytes 2026-02-20T21:55:44.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6282061824 bytes 2026-02-20T21:55:44.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6360451072 bytes 2026-02-20T21:55:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6397374464 bytes 2026-02-20T21:55:44.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6444470272 bytes 2026-02-20T21:55:44.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6536786944 bytes 2026-02-20T21:55:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6601966592 bytes 2026-02-20T21:55:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6649622528 bytes 2026-02-20T21:55:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6777547776 bytes 2026-02-20T21:55:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6822293504 bytes 2026-02-20T21:55:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6843398144 bytes 2026-02-20T21:55:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6860084224 bytes 2026-02-20T21:55:44.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6870029312 bytes 2026-02-20T21:55:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6870670336 bytes 2026-02-20T21:55:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6951575552 bytes 2026-02-20T21:55:44.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6957383680 bytes 2026-02-20T21:55:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7023561728 bytes 2026-02-20T21:55:45.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7067215872 bytes 2026-02-20T21:55:45.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7151276032 bytes 2026-02-20T21:55:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7186341888 bytes 2026-02-20T21:55:45.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7255559168 bytes 2026-02-20T21:55:45.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7294073856 bytes 2026-02-20T21:55:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7347828736 bytes 2026-02-20T21:55:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7403852800 bytes 2026-02-20T21:55:45.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7452990464 bytes 2026-02-20T21:55:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7493273600 bytes 2026-02-20T21:55:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7576871936 bytes 2026-02-20T21:55:45.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7597377536 bytes 2026-02-20T21:55:45.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7638021120 bytes 2026-02-20T21:55:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7638629376 bytes 2026-02-20T21:55:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7675345920 bytes 2026-02-20T21:55:45.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7680252928 bytes 2026-02-20T21:55:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7762367488 bytes 2026-02-20T21:55:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7838385152 bytes 2026-02-20T21:55:45.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7937207296 bytes 2026-02-20T21:55:46.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7989790720 bytes 2026-02-20T21:55:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7999474688 bytes 2026-02-20T21:55:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8014375936 bytes 2026-02-20T21:55:46.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8018139136 bytes 2026-02-20T21:55:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8029657088 bytes 2026-02-20T21:55:46.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8037925888 bytes 2026-02-20T21:55:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8042931200 bytes 2026-02-20T21:55:46.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8104540160 bytes 2026-02-20T21:55:46.340 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8148546560 bytes 2026-02-20T21:55:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8180547584 bytes 2026-02-20T21:55:46.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8219773952 bytes 2026-02-20T21:55:46.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8220379136 bytes 2026-02-20T21:55:46.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8282082304 bytes 2026-02-20T21:55:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:55:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:55:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:55:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:55:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_48 2026-02-20T21:55:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 snapshot 128M 2026-02-20T21:55:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_48 128M 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:46.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:46.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T21:55:46.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:46.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:55:46.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:55:46.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:55:46.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:55:46.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_48 2026-02-20T21:55:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:55:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:55:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:55:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_48 snapshot 2026-02-20T21:55:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:46.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T21:55:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:55:46.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_48 snapshot 2026-02-20T21:55:47.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:55:47.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_48 2026-02-20T21:55:47.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_48': 2026-02-20T21:55:47.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:55:47.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:55:47.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:55:47.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 14cd47f9e27d 2026-02-20T21:55:47.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.14cd47f9e27d 2026-02-20T21:55:47.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:55:47.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:55:47.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:55:47.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:55:47.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:46 2026 2026-02-20T21:55:47.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:46 2026 2026-02-20T21:55:47.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:46 2026 2026-02-20T21:55:47.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:55:47.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:55:47.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4a44519a-03e3-4a42-bf80-6ed4a8037a65 2026-02-20T21:55:47.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:55:47.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:55:47.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_48 2026-02-20T21:55:47.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:55:47.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:55:47.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T21:55:47.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:55:47.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=13 2026-02-20T21:55:47.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:55:47.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:55:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 73067520 bytes 2026-02-20T21:55:47.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 73698304 bytes 2026-02-20T21:55:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 157553664 bytes 2026-02-20T21:55:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 189812736 bytes 2026-02-20T21:55:47.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 200773632 bytes 2026-02-20T21:55:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 270361600 bytes 2026-02-20T21:55:47.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 324584448 bytes 2026-02-20T21:55:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 412668928 bytes 2026-02-20T21:55:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 446162944 bytes 2026-02-20T21:55:48.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 508827648 bytes 2026-02-20T21:55:48.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 599987200 bytes 2026-02-20T21:55:48.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 644575232 bytes 2026-02-20T21:55:48.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 731941888 bytes 2026-02-20T21:55:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 734625792 bytes 2026-02-20T21:55:48.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 758483968 bytes 2026-02-20T21:55:48.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 794947584 bytes 2026-02-20T21:55:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 853855232 bytes 2026-02-20T21:55:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 887977984 bytes 2026-02-20T21:55:48.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 918761472 bytes 2026-02-20T21:55:48.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 964452352 bytes 2026-02-20T21:55:48.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1007327232 bytes 2026-02-20T21:55:48.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1034560512 bytes 2026-02-20T21:55:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1055207424 bytes 2026-02-20T21:55:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1077093376 bytes 2026-02-20T21:55:48.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1106092032 bytes 2026-02-20T21:55:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1135519744 bytes 2026-02-20T21:55:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1180590080 bytes 2026-02-20T21:55:48.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1230934016 bytes 2026-02-20T21:55:49.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1231566848 bytes 2026-02-20T21:55:49.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1272915968 bytes 2026-02-20T21:55:49.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1315832832 bytes 2026-02-20T21:55:49.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1339641856 bytes 2026-02-20T21:55:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1421469696 bytes 2026-02-20T21:55:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1468305408 bytes 2026-02-20T21:55:49.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1505046528 bytes 2026-02-20T21:55:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1557042176 bytes 2026-02-20T21:55:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1605475328 bytes 2026-02-20T21:55:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1646256128 bytes 2026-02-20T21:55:49.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1650295808 bytes 2026-02-20T21:55:49.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1718258688 bytes 2026-02-20T21:55:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1777050624 bytes 2026-02-20T21:55:49.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1819539456 bytes 2026-02-20T21:55:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1848622080 bytes 2026-02-20T21:55:49.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1918397440 bytes 2026-02-20T21:55:49.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1938472960 bytes 2026-02-20T21:55:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1989751808 bytes 2026-02-20T21:55:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2063659008 bytes 2026-02-20T21:55:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2100224000 bytes 2026-02-20T21:55:49.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2161850368 bytes 2026-02-20T21:55:50.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2257240064 bytes 2026-02-20T21:55:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2267294720 bytes 2026-02-20T21:55:50.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2327230464 bytes 2026-02-20T21:55:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2347490304 bytes 2026-02-20T21:55:50.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2349083648 bytes 2026-02-20T21:55:50.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2371192832 bytes 2026-02-20T21:55:50.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2400715776 bytes 2026-02-20T21:55:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2405134336 bytes 2026-02-20T21:55:50.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2468041728 bytes 2026-02-20T21:55:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2498836480 bytes 2026-02-20T21:55:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2549007360 bytes 2026-02-20T21:55:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2564729856 bytes 2026-02-20T21:55:50.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2592420864 bytes 2026-02-20T21:55:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2652725248 bytes 2026-02-20T21:55:50.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2664523776 bytes 2026-02-20T21:55:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2692496384 bytes 2026-02-20T21:55:50.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2736314368 bytes 2026-02-20T21:55:50.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2773211136 bytes 2026-02-20T21:55:50.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2795597824 bytes 2026-02-20T21:55:50.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2828769280 bytes 2026-02-20T21:55:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2859242496 bytes 2026-02-20T21:55:50.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2917616640 bytes 2026-02-20T21:55:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2985209856 bytes 2026-02-20T21:55:51.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3022653440 bytes 2026-02-20T21:55:51.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3040654336 bytes 2026-02-20T21:55:51.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3084294144 bytes 2026-02-20T21:55:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3098926080 bytes 2026-02-20T21:55:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3125545984 bytes 2026-02-20T21:55:51.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3126165504 bytes 2026-02-20T21:55:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3178067968 bytes 2026-02-20T21:55:51.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3301186560 bytes 2026-02-20T21:55:51.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3357139968 bytes 2026-02-20T21:55:51.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3411562496 bytes 2026-02-20T21:55:51.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3454336000 bytes 2026-02-20T21:55:51.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3495573504 bytes 2026-02-20T21:55:51.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3535005696 bytes 2026-02-20T21:55:51.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3543681024 bytes 2026-02-20T21:55:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3609183232 bytes 2026-02-20T21:55:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3685849088 bytes 2026-02-20T21:55:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3701430272 bytes 2026-02-20T21:55:51.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3712193536 bytes 2026-02-20T21:55:51.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3712803840 bytes 2026-02-20T21:55:51.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3713412096 bytes 2026-02-20T21:55:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3789257728 bytes 2026-02-20T21:55:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3799784448 bytes 2026-02-20T21:55:52.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3859368960 bytes 2026-02-20T21:55:52.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3937671168 bytes 2026-02-20T21:55:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4011319296 bytes 2026-02-20T21:55:52.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4044348416 bytes 2026-02-20T21:55:52.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4062142464 bytes 2026-02-20T21:55:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4102318080 bytes 2026-02-20T21:55:52.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4108388352 bytes 2026-02-20T21:55:52.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4155275264 bytes 2026-02-20T21:55:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4233600000 bytes 2026-02-20T21:55:52.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4274191360 bytes 2026-02-20T21:55:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4291850240 bytes 2026-02-20T21:55:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4362791936 bytes 2026-02-20T21:55:52.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4408448000 bytes 2026-02-20T21:55:52.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4454841344 bytes 2026-02-20T21:55:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4505921536 bytes 2026-02-20T21:55:52.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4556590080 bytes 2026-02-20T21:55:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4633113600 bytes 2026-02-20T21:55:52.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4659649536 bytes 2026-02-20T21:55:52.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4701928448 bytes 2026-02-20T21:55:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4709206016 bytes 2026-02-20T21:55:53.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4724357120 bytes 2026-02-20T21:55:53.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4788397056 bytes 2026-02-20T21:55:53.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4829565952 bytes 2026-02-20T21:55:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4866008064 bytes 2026-02-20T21:55:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4902297600 bytes 2026-02-20T21:55:53.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4977780736 bytes 2026-02-20T21:55:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5065933824 bytes 2026-02-20T21:55:53.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5123688448 bytes 2026-02-20T21:55:53.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5133461504 bytes 2026-02-20T21:55:53.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5134084096 bytes 2026-02-20T21:55:53.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5169745920 bytes 2026-02-20T21:55:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5222202368 bytes 2026-02-20T21:55:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5271733248 bytes 2026-02-20T21:55:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5297800192 bytes 2026-02-20T21:55:53.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5385050112 bytes 2026-02-20T21:55:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5421699072 bytes 2026-02-20T21:55:53.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5493377024 bytes 2026-02-20T21:55:53.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5524367360 bytes 2026-02-20T21:55:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5586533376 bytes 2026-02-20T21:55:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5593914368 bytes 2026-02-20T21:55:54.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5614926848 bytes 2026-02-20T21:55:54.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5698636800 bytes 2026-02-20T21:55:54.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5740085248 bytes 2026-02-20T21:55:54.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5778664448 bytes 2026-02-20T21:55:54.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5865810944 bytes 2026-02-20T21:55:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5932303360 bytes 2026-02-20T21:55:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5994867712 bytes 2026-02-20T21:55:54.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6017585152 bytes 2026-02-20T21:55:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6077580288 bytes 2026-02-20T21:55:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6151205888 bytes 2026-02-20T21:55:54.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6178129920 bytes 2026-02-20T21:55:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6190448640 bytes 2026-02-20T21:55:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6200017920 bytes 2026-02-20T21:55:54.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6250220544 bytes 2026-02-20T21:55:54.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6346184704 bytes 2026-02-20T21:55:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6402243584 bytes 2026-02-20T21:55:54.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6402844672 bytes 2026-02-20T21:55:54.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6433157120 bytes 2026-02-20T21:55:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6507851776 bytes 2026-02-20T21:55:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6554826752 bytes 2026-02-20T21:55:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6603646976 bytes 2026-02-20T21:55:55.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6622879744 bytes 2026-02-20T21:55:55.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6683352064 bytes 2026-02-20T21:55:55.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6753117184 bytes 2026-02-20T21:55:55.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6786052096 bytes 2026-02-20T21:55:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6831949824 bytes 2026-02-20T21:55:55.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6860852224 bytes 2026-02-20T21:55:55.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6875848704 bytes 2026-02-20T21:55:55.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6948632576 bytes 2026-02-20T21:55:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6991514624 bytes 2026-02-20T21:55:55.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7016995840 bytes 2026-02-20T21:55:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7069332480 bytes 2026-02-20T21:55:55.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7095810048 bytes 2026-02-20T21:55:55.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7126452224 bytes 2026-02-20T21:55:55.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7232550912 bytes 2026-02-20T21:55:55.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7244925952 bytes 2026-02-20T21:55:55.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7267613696 bytes 2026-02-20T21:55:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7309503488 bytes 2026-02-20T21:55:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7345961984 bytes 2026-02-20T21:55:55.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7421575168 bytes 2026-02-20T21:55:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7444170752 bytes 2026-02-20T21:55:56.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7519708160 bytes 2026-02-20T21:55:56.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7586092032 bytes 2026-02-20T21:55:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7586717696 bytes 2026-02-20T21:55:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7647068160 bytes 2026-02-20T21:55:56.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7684998144 bytes 2026-02-20T21:55:56.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7781795840 bytes 2026-02-20T21:55:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7820096512 bytes 2026-02-20T21:55:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7826908160 bytes 2026-02-20T21:55:56.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7858012160 bytes 2026-02-20T21:55:56.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7945967616 bytes 2026-02-20T21:55:56.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7973651456 bytes 2026-02-20T21:55:56.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8012214272 bytes 2026-02-20T21:55:56.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8090432512 bytes 2026-02-20T21:55:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8091359232 bytes 2026-02-20T21:55:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8166041600 bytes 2026-02-20T21:55:56.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8190995456 bytes 2026-02-20T21:55:56.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8203020288 bytes 2026-02-20T21:55:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8286352384 bytes 2026-02-20T21:55:56.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8318183424 bytes 2026-02-20T21:55:56.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8340754432 bytes 2026-02-20T21:55:56.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8378906624 bytes 2026-02-20T21:55:57.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8398607360 bytes 2026-02-20T21:55:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8399195136 bytes 2026-02-20T21:55:57.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8410586112 bytes 2026-02-20T21:55:57.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8429388800 bytes 2026-02-20T21:55:57.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8445778944 bytes 2026-02-20T21:55:57.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8450080768 bytes 2026-02-20T21:55:57.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8503113728 bytes 2026-02-20T21:55:57.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8521964544 bytes 2026-02-20T21:55:57.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8625256448 bytes 2026-02-20T21:55:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8631140352 bytes 2026-02-20T21:55:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8755521536 bytes 2026-02-20T21:55:57.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8814769152 bytes 2026-02-20T21:55:57.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8824819712 bytes 2026-02-20T21:55:57.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8841909248 bytes 2026-02-20T21:55:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8845145088 bytes 2026-02-20T21:55:57.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8848487424 bytes 2026-02-20T21:55:57.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8903960576 bytes 2026-02-20T21:55:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8938916864 bytes 2026-02-20T21:55:57.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8959196160 bytes 2026-02-20T21:55:57.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 8973673472 bytes 2026-02-20T21:55:57.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 8996969472 bytes 2026-02-20T21:55:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9011963904 bytes 2026-02-20T21:55:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9035527168 bytes 2026-02-20T21:55:57.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9066221568 bytes 2026-02-20T21:55:57.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9145551872 bytes 2026-02-20T21:55:58.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9226054656 bytes 2026-02-20T21:55:58.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9250722816 bytes 2026-02-20T21:55:58.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9283831808 bytes 2026-02-20T21:55:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9350065152 bytes 2026-02-20T21:55:58.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9354138624 bytes 2026-02-20T21:55:58.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9414736896 bytes 2026-02-20T21:55:58.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9430592512 bytes 2026-02-20T21:55:58.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9504629760 bytes 2026-02-20T21:55:58.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9590105088 bytes 2026-02-20T21:55:58.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9622102016 bytes 2026-02-20T21:55:58.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9648663552 bytes 2026-02-20T21:55:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9649275904 bytes 2026-02-20T21:55:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9650658304 bytes 2026-02-20T21:55:58.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9705959424 bytes 2026-02-20T21:55:58.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 9797689344 bytes 2026-02-20T21:55:58.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 9837902848 bytes 2026-02-20T21:55:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 9863643136 bytes 2026-02-20T21:55:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 9930949632 bytes 2026-02-20T21:55:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 9938528256 bytes 2026-02-20T21:55:58.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 9958116352 bytes 2026-02-20T21:55:58.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 9981487104 bytes 2026-02-20T21:55:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 9992122368 bytes 2026-02-20T21:55:59.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 10010387456 bytes 2026-02-20T21:55:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10083018752 bytes 2026-02-20T21:55:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10151830528 bytes 2026-02-20T21:55:59.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10232478720 bytes 2026-02-20T21:55:59.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10266287104 bytes 2026-02-20T21:55:59.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 10309337088 bytes 2026-02-20T21:55:59.359 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 10354049024 bytes 2026-02-20T21:55:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25100 IOs, 10359864320 bytes 2026-02-20T21:55:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25200 IOs, 10381993984 bytes 2026-02-20T21:55:59.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25300 IOs, 10452023296 bytes 2026-02-20T21:55:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25400 IOs, 10539287552 bytes 2026-02-20T21:55:59.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25500 IOs, 10593097728 bytes 2026-02-20T21:55:59.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25600 IOs, 10593716224 bytes 2026-02-20T21:55:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25700 IOs, 10677228544 bytes 2026-02-20T21:55:59.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25800 IOs, 10703084544 bytes 2026-02-20T21:55:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25900 IOs, 10736524288 bytes 2026-02-20T21:55:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26000 IOs, 10762782720 bytes 2026-02-20T21:55:59.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26100 IOs, 10788842496 bytes 2026-02-20T21:55:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26200 IOs, 10843580416 bytes 2026-02-20T21:55:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26300 IOs, 10853920768 bytes 2026-02-20T21:56:00.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26400 IOs, 10913505280 bytes 2026-02-20T21:56:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26500 IOs, 10944910336 bytes 2026-02-20T21:56:00.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26600 IOs, 10996226048 bytes 2026-02-20T21:56:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26700 IOs, 11012541440 bytes 2026-02-20T21:56:00.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26800 IOs, 11079808000 bytes 2026-02-20T21:56:00.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26900 IOs, 11098870784 bytes 2026-02-20T21:56:00.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27000 IOs, 11114075136 bytes 2026-02-20T21:56:00.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27100 IOs, 11185720320 bytes 2026-02-20T21:56:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27200 IOs, 11301687296 bytes 2026-02-20T21:56:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27300 IOs, 11317996544 bytes 2026-02-20T21:56:00.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27400 IOs, 11375745024 bytes 2026-02-20T21:56:00.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:27500 IOs, 11437255680 bytes 2026-02-20T21:56:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:56:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:56:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:56:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:56:00.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_49 2026-02-20T21:56:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 snapshot 128M 2026-02-20T21:56:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T21:56:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:56:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_49 128M 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:56:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_49 2026-02-20T21:56:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:56:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:56:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_49 snapshot 2026-02-20T21:56:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T21:56:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:56:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_49 snapshot 2026-02-20T21:56:01.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:56:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_49 2026-02-20T21:56:01.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_49': 2026-02-20T21:56:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:56:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:56:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:56:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 14d6320e821a 2026-02-20T21:56:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.14d6320e821a 2026-02-20T21:56:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:56:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:56:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:56:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:56:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:56:00 2026 2026-02-20T21:56:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:56:00 2026 2026-02-20T21:56:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:56:00 2026 2026-02-20T21:56:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:56:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:56:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: d0428f4e-f589-457e-8185-f6b0bba89f12 2026-02-20T21:56:01.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:56:01.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:56:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_49 2026-02-20T21:56:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T21:56:01.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:56:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=7 2026-02-20T21:56:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:56:01.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:56:01.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 52919296 bytes 2026-02-20T21:56:01.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 79007744 bytes 2026-02-20T21:56:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 161521664 bytes 2026-02-20T21:56:01.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 224543744 bytes 2026-02-20T21:56:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 239379456 bytes 2026-02-20T21:56:01.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 322550784 bytes 2026-02-20T21:56:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 362525696 bytes 2026-02-20T21:56:02.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 406364160 bytes 2026-02-20T21:56:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 471768064 bytes 2026-02-20T21:56:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 549369856 bytes 2026-02-20T21:56:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 595492864 bytes 2026-02-20T21:56:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 656492544 bytes 2026-02-20T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 696804352 bytes 2026-02-20T21:56:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 722437120 bytes 2026-02-20T21:56:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 739330048 bytes 2026-02-20T21:56:02.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 744214528 bytes 2026-02-20T21:56:02.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 761063424 bytes 2026-02-20T21:56:02.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 785423360 bytes 2026-02-20T21:56:02.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 833378304 bytes 2026-02-20T21:56:02.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 888407040 bytes 2026-02-20T21:56:02.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 941813760 bytes 2026-02-20T21:56:02.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 996933632 bytes 2026-02-20T21:56:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1075432448 bytes 2026-02-20T21:56:02.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1170252800 bytes 2026-02-20T21:56:02.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1232586752 bytes 2026-02-20T21:56:02.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1252027392 bytes 2026-02-20T21:56:02.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1315772416 bytes 2026-02-20T21:56:03.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1390021632 bytes 2026-02-20T21:56:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1433758720 bytes 2026-02-20T21:56:03.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1479299072 bytes 2026-02-20T21:56:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1479914496 bytes 2026-02-20T21:56:03.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1558078464 bytes 2026-02-20T21:56:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1632740352 bytes 2026-02-20T21:56:03.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1676196864 bytes 2026-02-20T21:56:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1719909376 bytes 2026-02-20T21:56:03.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1792724992 bytes 2026-02-20T21:56:03.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1873883136 bytes 2026-02-20T21:56:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1936138240 bytes 2026-02-20T21:56:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1942201344 bytes 2026-02-20T21:56:03.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 2054614016 bytes 2026-02-20T21:56:03.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 2069970944 bytes 2026-02-20T21:56:03.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2117542912 bytes 2026-02-20T21:56:03.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2158973952 bytes 2026-02-20T21:56:03.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2176459776 bytes 2026-02-20T21:56:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2243577856 bytes 2026-02-20T21:56:03.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2271365120 bytes 2026-02-20T21:56:04.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2296378368 bytes 2026-02-20T21:56:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2311606272 bytes 2026-02-20T21:56:04.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2340036608 bytes 2026-02-20T21:56:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2358850560 bytes 2026-02-20T21:56:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2387012608 bytes 2026-02-20T21:56:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2426292224 bytes 2026-02-20T21:56:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2465501184 bytes 2026-02-20T21:56:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2562140160 bytes 2026-02-20T21:56:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2595859456 bytes 2026-02-20T21:56:04.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2596988928 bytes 2026-02-20T21:56:04.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2713977856 bytes 2026-02-20T21:56:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2723589120 bytes 2026-02-20T21:56:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2733748224 bytes 2026-02-20T21:56:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2773720064 bytes 2026-02-20T21:56:04.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2803491840 bytes 2026-02-20T21:56:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2827872256 bytes 2026-02-20T21:56:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2867782656 bytes 2026-02-20T21:56:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2869970944 bytes 2026-02-20T21:56:04.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2875336704 bytes 2026-02-20T21:56:04.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2947558400 bytes 2026-02-20T21:56:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3017194496 bytes 2026-02-20T21:56:04.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3097554944 bytes 2026-02-20T21:56:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3136940032 bytes 2026-02-20T21:56:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3151482880 bytes 2026-02-20T21:56:05.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3167407104 bytes 2026-02-20T21:56:05.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3171614720 bytes 2026-02-20T21:56:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3255365632 bytes 2026-02-20T21:56:05.274 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3340008448 bytes 2026-02-20T21:56:05.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3427258368 bytes 2026-02-20T21:56:05.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3478998016 bytes 2026-02-20T21:56:05.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3547010048 bytes 2026-02-20T21:56:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3569867776 bytes 2026-02-20T21:56:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3617022976 bytes 2026-02-20T21:56:05.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3675563008 bytes 2026-02-20T21:56:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3722819584 bytes 2026-02-20T21:56:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3748093952 bytes 2026-02-20T21:56:05.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3814078464 bytes 2026-02-20T21:56:06.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3884939264 bytes 2026-02-20T21:56:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3923286016 bytes 2026-02-20T21:56:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3935593472 bytes 2026-02-20T21:56:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3971579904 bytes 2026-02-20T21:56:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4025504768 bytes 2026-02-20T21:56:06.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4094194688 bytes 2026-02-20T21:56:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4182054912 bytes 2026-02-20T21:56:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4245055488 bytes 2026-02-20T21:56:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4351956992 bytes 2026-02-20T21:56:06.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4369161216 bytes 2026-02-20T21:56:06.522 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4384625664 bytes 2026-02-20T21:56:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4394102784 bytes 2026-02-20T21:56:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4459831296 bytes 2026-02-20T21:56:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4464063488 bytes 2026-02-20T21:56:06.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4535600128 bytes 2026-02-20T21:56:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4565761024 bytes 2026-02-20T21:56:06.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4602446848 bytes 2026-02-20T21:56:06.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4657850368 bytes 2026-02-20T21:56:06.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4683827200 bytes 2026-02-20T21:56:06.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4790214656 bytes 2026-02-20T21:56:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4874544128 bytes 2026-02-20T21:56:07.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4915176448 bytes 2026-02-20T21:56:07.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4945587200 bytes 2026-02-20T21:56:07.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 5010037760 bytes 2026-02-20T21:56:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 5087940608 bytes 2026-02-20T21:56:07.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 5140375552 bytes 2026-02-20T21:56:07.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5189773312 bytes 2026-02-20T21:56:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5275002880 bytes 2026-02-20T21:56:07.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5330497536 bytes 2026-02-20T21:56:07.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5372119040 bytes 2026-02-20T21:56:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5434348544 bytes 2026-02-20T21:56:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5455324160 bytes 2026-02-20T21:56:07.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5553956864 bytes 2026-02-20T21:56:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5605170176 bytes 2026-02-20T21:56:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5616619520 bytes 2026-02-20T21:56:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5666211840 bytes 2026-02-20T21:56:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5687763968 bytes 2026-02-20T21:56:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5707312128 bytes 2026-02-20T21:56:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5767789568 bytes 2026-02-20T21:56:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5820812288 bytes 2026-02-20T21:56:08.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5835715584 bytes 2026-02-20T21:56:08.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5865010176 bytes 2026-02-20T21:56:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5904054272 bytes 2026-02-20T21:56:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5959223296 bytes 2026-02-20T21:56:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5976375296 bytes 2026-02-20T21:56:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 6017594368 bytes 2026-02-20T21:56:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 6085855232 bytes 2026-02-20T21:56:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 6119640064 bytes 2026-02-20T21:56:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 6135851008 bytes 2026-02-20T21:56:08.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 6209012736 bytes 2026-02-20T21:56:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 6251746304 bytes 2026-02-20T21:56:08.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 6297059328 bytes 2026-02-20T21:56:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:56:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:56:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:56:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:56:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_50 2026-02-20T21:56:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 snapshot 128M 2026-02-20T21:56:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:08.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n snapshot ']' 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_50 128M 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T21:56:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T21:56:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 128M ']' 2026-02-20T21:56:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=128M 2026-02-20T21:56:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T21:56:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_50 2026-02-20T21:56:08.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:56:08.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:08.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' image = image ']' 2026-02-20T21:56:08.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:08.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_50 snapshot 2026-02-20T21:56:08.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T21:56:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T21:56:08.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_50 snapshot 2026-02-20T21:56:09.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T21:56:09.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_50 2026-02-20T21:56:09.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_50': 2026-02-20T21:56:09.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:56:09.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:56:09.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T21:56:09.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 14dff0e1a71f 2026-02-20T21:56:09.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.14dff0e1a71f 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:56:08 2026 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:56:08 2026 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:56:08 2026 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: f06fe263-7213-4c42-b77b-594ccc26a39b 2026-02-20T21:56:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T21:56:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n '' ']' 2026-02-20T21:56:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_50 2026-02-20T21:56:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T21:56:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:56:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=11 2026-02-20T21:56:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T21:56:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:56:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 42852352 bytes 2026-02-20T21:56:09.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 73306112 bytes 2026-02-20T21:56:09.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 118912000 bytes 2026-02-20T21:56:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 148409344 bytes 2026-02-20T21:56:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 185443328 bytes 2026-02-20T21:56:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 240615424 bytes 2026-02-20T21:56:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 309872640 bytes 2026-02-20T21:56:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 380126208 bytes 2026-02-20T21:56:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 454427648 bytes 2026-02-20T21:56:10.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 516739072 bytes 2026-02-20T21:56:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 564097024 bytes 2026-02-20T21:56:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 593025024 bytes 2026-02-20T21:56:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 706102272 bytes 2026-02-20T21:56:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 734117888 bytes 2026-02-20T21:56:10.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 785837056 bytes 2026-02-20T21:56:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 847237120 bytes 2026-02-20T21:56:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 898113536 bytes 2026-02-20T21:56:10.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 1003941888 bytes 2026-02-20T21:56:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 1006039040 bytes 2026-02-20T21:56:10.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1027825664 bytes 2026-02-20T21:56:10.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1045009408 bytes 2026-02-20T21:56:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1103348736 bytes 2026-02-20T21:56:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1151439872 bytes 2026-02-20T21:56:10.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1184358400 bytes 2026-02-20T21:56:10.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1212909568 bytes 2026-02-20T21:56:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1258543104 bytes 2026-02-20T21:56:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1277710336 bytes 2026-02-20T21:56:11.075 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1319248896 bytes 2026-02-20T21:56:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1374185472 bytes 2026-02-20T21:56:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1473461248 bytes 2026-02-20T21:56:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1532180480 bytes 2026-02-20T21:56:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1585889280 bytes 2026-02-20T21:56:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1650192384 bytes 2026-02-20T21:56:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1665476608 bytes 2026-02-20T21:56:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1695490048 bytes 2026-02-20T21:56:11.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1705985024 bytes 2026-02-20T21:56:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1800457216 bytes 2026-02-20T21:56:11.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1852386304 bytes 2026-02-20T21:56:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1883522048 bytes 2026-02-20T21:56:11.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1931482112 bytes 2026-02-20T21:56:11.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1968460800 bytes 2026-02-20T21:56:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1997845504 bytes 2026-02-20T21:56:11.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2075118592 bytes 2026-02-20T21:56:11.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2129115136 bytes 2026-02-20T21:56:11.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2147037184 bytes 2026-02-20T21:56:11.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2184519680 bytes 2026-02-20T21:56:12.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2270779392 bytes 2026-02-20T21:56:12.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2272400384 bytes 2026-02-20T21:56:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2325529600 bytes 2026-02-20T21:56:12.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2385239040 bytes 2026-02-20T21:56:12.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2443845632 bytes 2026-02-20T21:56:12.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2476297216 bytes 2026-02-20T21:56:12.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2570340352 bytes 2026-02-20T21:56:12.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2648693760 bytes 2026-02-20T21:56:12.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2662360064 bytes 2026-02-20T21:56:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2732323840 bytes 2026-02-20T21:56:12.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2747592704 bytes 2026-02-20T21:56:12.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2805987328 bytes 2026-02-20T21:56:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2812398592 bytes 2026-02-20T21:56:12.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2906045440 bytes 2026-02-20T21:56:12.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2940676096 bytes 2026-02-20T21:56:12.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2993722368 bytes 2026-02-20T21:56:12.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 3049470976 bytes 2026-02-20T21:56:12.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3091010560 bytes 2026-02-20T21:56:12.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3127784448 bytes 2026-02-20T21:56:13.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3225169920 bytes 2026-02-20T21:56:13.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3267717120 bytes 2026-02-20T21:56:13.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3286804480 bytes 2026-02-20T21:56:13.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3321107456 bytes 2026-02-20T21:56:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3328829440 bytes 2026-02-20T21:56:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3410010112 bytes 2026-02-20T21:56:13.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3425486848 bytes 2026-02-20T21:56:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3509952512 bytes 2026-02-20T21:56:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3542139904 bytes 2026-02-20T21:56:13.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3565534208 bytes 2026-02-20T21:56:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3592585216 bytes 2026-02-20T21:56:13.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3613589504 bytes 2026-02-20T21:56:13.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3614204928 bytes 2026-02-20T21:56:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3670781952 bytes 2026-02-20T21:56:13.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3684393984 bytes 2026-02-20T21:56:13.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3756561408 bytes 2026-02-20T21:56:13.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3811982336 bytes 2026-02-20T21:56:13.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3891037184 bytes 2026-02-20T21:56:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3906339840 bytes 2026-02-20T21:56:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3994503168 bytes 2026-02-20T21:56:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 4020102144 bytes 2026-02-20T21:56:14.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 4038526976 bytes 2026-02-20T21:56:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4078999552 bytes 2026-02-20T21:56:14.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4156762112 bytes 2026-02-20T21:56:14.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4190837760 bytes 2026-02-20T21:56:14.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4220043264 bytes 2026-02-20T21:56:14.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4249469952 bytes 2026-02-20T21:56:14.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4352759808 bytes 2026-02-20T21:56:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4379780096 bytes 2026-02-20T21:56:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4441969664 bytes 2026-02-20T21:56:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4487597056 bytes 2026-02-20T21:56:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4526094336 bytes 2026-02-20T21:56:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4572690432 bytes 2026-02-20T21:56:14.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4608389120 bytes 2026-02-20T21:56:15.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4638613504 bytes 2026-02-20T21:56:15.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4701896704 bytes 2026-02-20T21:56:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4749273088 bytes 2026-02-20T21:56:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4788639744 bytes 2026-02-20T21:56:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4812013568 bytes 2026-02-20T21:56:15.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4812643328 bytes 2026-02-20T21:56:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4862526464 bytes 2026-02-20T21:56:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4916804608 bytes 2026-02-20T21:56:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4993994752 bytes 2026-02-20T21:56:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 5026363392 bytes 2026-02-20T21:56:15.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5087620096 bytes 2026-02-20T21:56:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5132701696 bytes 2026-02-20T21:56:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5158470656 bytes 2026-02-20T21:56:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5234229248 bytes 2026-02-20T21:56:15.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5253035008 bytes 2026-02-20T21:56:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5288714240 bytes 2026-02-20T21:56:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5290729472 bytes 2026-02-20T21:56:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5341207552 bytes 2026-02-20T21:56:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5372153856 bytes 2026-02-20T21:56:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5429072896 bytes 2026-02-20T21:56:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5517618176 bytes 2026-02-20T21:56:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5555822592 bytes 2026-02-20T21:56:16.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5615864832 bytes 2026-02-20T21:56:16.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5722407936 bytes 2026-02-20T21:56:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5749935104 bytes 2026-02-20T21:56:16.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5799243776 bytes 2026-02-20T21:56:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5848576000 bytes 2026-02-20T21:56:16.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5885726720 bytes 2026-02-20T21:56:16.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5935486976 bytes 2026-02-20T21:56:16.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5966606336 bytes 2026-02-20T21:56:16.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5978974208 bytes 2026-02-20T21:56:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 6014735360 bytes 2026-02-20T21:56:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 6085403648 bytes 2026-02-20T21:56:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 6141077504 bytes 2026-02-20T21:56:16.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 6180986880 bytes 2026-02-20T21:56:16.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 6200860672 bytes 2026-02-20T21:56:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 6358683648 bytes 2026-02-20T21:56:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 6377140224 bytes 2026-02-20T21:56:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6377769984 bytes 2026-02-20T21:56:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6387170304 bytes 2026-02-20T21:56:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6399262720 bytes 2026-02-20T21:56:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6468803584 bytes 2026-02-20T21:56:17.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6512707584 bytes 2026-02-20T21:56:17.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6599794688 bytes 2026-02-20T21:56:17.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6638154752 bytes 2026-02-20T21:56:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6678437888 bytes 2026-02-20T21:56:17.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6724515840 bytes 2026-02-20T21:56:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6738679808 bytes 2026-02-20T21:56:17.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6764981248 bytes 2026-02-20T21:56:17.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6765599744 bytes 2026-02-20T21:56:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6806336512 bytes 2026-02-20T21:56:17.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6888919040 bytes 2026-02-20T21:56:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6908815360 bytes 2026-02-20T21:56:17.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6984382464 bytes 2026-02-20T21:56:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 7020173312 bytes 2026-02-20T21:56:17.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 7030628352 bytes 2026-02-20T21:56:17.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 7078394880 bytes 2026-02-20T21:56:17.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 7124025344 bytes 2026-02-20T21:56:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 7173145600 bytes 2026-02-20T21:56:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 7198769152 bytes 2026-02-20T21:56:18.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 7298608128 bytes 2026-02-20T21:56:18.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 7317131264 bytes 2026-02-20T21:56:18.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 7331000320 bytes 2026-02-20T21:56:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7348246528 bytes 2026-02-20T21:56:18.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7370471424 bytes 2026-02-20T21:56:18.340 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7434973184 bytes 2026-02-20T21:56:18.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7473741824 bytes 2026-02-20T21:56:18.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7501410304 bytes 2026-02-20T21:56:18.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7555248128 bytes 2026-02-20T21:56:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7589740544 bytes 2026-02-20T21:56:18.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7607319552 bytes 2026-02-20T21:56:18.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7712897024 bytes 2026-02-20T21:56:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7765964800 bytes 2026-02-20T21:56:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7840620544 bytes 2026-02-20T21:56:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7855418368 bytes 2026-02-20T21:56:18.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7939233792 bytes 2026-02-20T21:56:18.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7957143552 bytes 2026-02-20T21:56:18.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 8019656704 bytes 2026-02-20T21:56:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 8067208192 bytes 2026-02-20T21:56:19.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 8104342528 bytes 2026-02-20T21:56:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 8162396160 bytes 2026-02-20T21:56:19.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 8226562048 bytes 2026-02-20T21:56:19.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 8274024448 bytes 2026-02-20T21:56:19.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 8305402880 bytes 2026-02-20T21:56:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 8343366656 bytes 2026-02-20T21:56:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 8399062016 bytes 2026-02-20T21:56:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 8431111168 bytes 2026-02-20T21:56:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8472212480 bytes 2026-02-20T21:56:19.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8507951104 bytes 2026-02-20T21:56:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8521290752 bytes 2026-02-20T21:56:19.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8582264832 bytes 2026-02-20T21:56:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8673979392 bytes 2026-02-20T21:56:19.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8740505600 bytes 2026-02-20T21:56:19.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8754984960 bytes 2026-02-20T21:56:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8818981888 bytes 2026-02-20T21:56:19.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8841089024 bytes 2026-02-20T21:56:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8841691136 bytes 2026-02-20T21:56:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8901251072 bytes 2026-02-20T21:56:19.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8904814592 bytes 2026-02-20T21:56:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8986050560 bytes 2026-02-20T21:56:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 9050718208 bytes 2026-02-20T21:56:20.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 9099153408 bytes 2026-02-20T21:56:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 9162301440 bytes 2026-02-20T21:56:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 9188100096 bytes 2026-02-20T21:56:20.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 9223969792 bytes 2026-02-20T21:56:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 9235403776 bytes 2026-02-20T21:56:20.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 9242782720 bytes 2026-02-20T21:56:20.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 9293608960 bytes 2026-02-20T21:56:20.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 9357859840 bytes 2026-02-20T21:56:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 9358479360 bytes 2026-02-20T21:56:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9401586688 bytes 2026-02-20T21:56:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_pool_images cluster2 mirror 50 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_count=50 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local count 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_count=0 2026-02-20T21:56:20.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:56:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 40 2026-02-20T21:56:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in `seq 1 40` 2026-02-20T21:56:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 1 -ne 1 2026-02-20T21:56:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-20T21:56:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep 'images: ' 2026-02-20T21:56:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cut '-d ' -f 2 2026-02-20T21:56:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:56:20.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ count=50 2026-02-20T21:56:20.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 50 = 50 2026-02-20T21:56:20.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_pool_healthy cluster2 mirror 2026-02-20T21:56:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:56:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state 2026-02-20T21:56:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 40 2026-02-20T21:56:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in `seq 1 40` 2026-02-20T21:56:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 1 -ne 1 2026-02-20T21:56:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-20T21:56:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep 'image health:' 2026-02-20T21:56:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cut '-d ' -f 3 2026-02-20T21:56:20.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:56:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=OK 2026-02-20T21:56:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test OK = ERROR 2026-02-20T21:56:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test OK = OK 2026-02-20T21:56:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_pool_images cluster1 mirror 50 2026-02-20T21:56:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_count=50 2026-02-20T21:56:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:56:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local count 2026-02-20T21:56:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_count=0 2026-02-20T21:56:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:56:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 40 2026-02-20T21:56:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in `seq 1 40` 2026-02-20T21:56:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 1 -ne 1 2026-02-20T21:56:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T21:56:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep 'images: ' 2026-02-20T21:56:20.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cut '-d ' -f 2 2026-02-20T21:56:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:56:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ count=50 2026-02-20T21:56:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 50 = 50 2026-02-20T21:56:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_pool_healthy cluster1 mirror 2026-02-20T21:56:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:56:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state 2026-02-20T21:56:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 40 2026-02-20T21:56:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in `seq 1 40` 2026-02-20T21:56:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 1 -ne 1 2026-02-20T21:56:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T21:56:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep 'image health:' 2026-02-20T21:56:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cut '-d ' -f 3 2026-02-20T21:56:20.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:56:20.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=WARNING 2026-02-20T21:56:20.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test WARNING = ERROR 2026-02-20T21:56:20.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test WARNING = OK 2026-02-20T21:56:20.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in `seq 1 40` 2026-02-20T21:56:20.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 2 -ne 1 2026-02-20T21:56:20.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 30 2026-02-20T21:56:28.557 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5893775202177007) > weight (0.25) 2026-02-20T21:56:28.557 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9606384526661285) > weight (0.25) 2026-02-20T21:56:28.557 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-20T21:56:28.557 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-20T21:56:28.558 INFO:tasks.rbd_mirror_thrash:waiting for 0.2954392775579362 secs before reviving daemons 2026-02-20T21:56:28.558 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T21:56:28.554+0000 7fd718b8e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 22748) UID: 1000 2026-02-20T21:56:28.854 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-20T21:56:28.854 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:56:28.854 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-20T21:56:28.854 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-20T21:56:28.855 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-20T21:56:28.857 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-20T21:56:28.857 INFO:tasks.rbd_mirror_thrash:waiting for 117.55655723540774 secs before thrashing 2026-02-20T21:56:50.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T21:56:50.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep 'image health:' 2026-02-20T21:56:50.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cut '-d ' -f 3 2026-02-20T21:56:50.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:56:50.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=OK 2026-02-20T21:56:50.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test OK = ERROR 2026-02-20T21:56:50.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test OK = OK 2026-02-20T21:56:50.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:50.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ testlog 'TEST: compare many images' 2026-02-20T21:56:50.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ tee -a /tmp/tmp.2SC0OFePO0/rbd-mirror.test.log 2026-02-20T21:56:50.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ date '+%F %T' 2026-02-20T21:56:50.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo 2026-02-20 21:56:50 TEST: compare many images 2026-02-20T21:56:50.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:2026-02-20 21:56:50 TEST: compare many images 2026-02-20T21:56:50.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 50 2026-02-20T21:56:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:56:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_1 2026-02-20T21:56:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_1 snap 2026-02-20T21:56:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:56:50.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:56:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2026-02-20T21:56:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2026-02-20T21:56:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:56:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:56:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:56:52.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:56:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2026-02-20T21:56:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:56:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:56:53.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:56:53.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2026-02-20T21:56:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:56:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:56:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:56:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:56:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:56:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:56:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:56:53.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:56:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:56:53.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:56:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:56:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2026-02-20T21:56:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:56:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:56:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:56:53.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:56:53.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:56:53.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:56:53.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:56:53.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:56:53.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:56:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:56:53.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:56:53.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:56:53.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2026-02-20T21:56:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2026-02-20T21:56:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:56:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:56:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:56:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2026-02-20T21:56:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:56:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:56:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:56:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:56:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:56:53.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:56:53.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:56:53.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:56:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:56:53.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:56:53.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:56:53.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:56:53.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:56:53.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2026-02-20T21:56:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2026-02-20T21:56:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:56:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:56:53.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:56:53.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2026-02-20T21:56:53.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:56:53.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:56:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:56:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:56:53.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:56:53.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:56:53.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:56:53.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:56:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:56:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:56:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:56:53.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2026-02-20T21:56:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_1'] 2026-02-20T21:56:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:56:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:56:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:56:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_1 2026-02-20T21:56:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:56:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:56:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:56:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:56:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:56:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:56:53.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:56:53.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:56:53.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:56:53.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:56:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:56:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_1 2026-02-20T21:56:53.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:56:53.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:56:53.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:56:53.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:56:53.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:56:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:56:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_1 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_1 2026-02-20T21:56:53.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:53.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_1 2026-02-20T21:56:54.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 86 2026-02-20T21:56:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_1 2026-02-20T21:56:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:56:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:56:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:56:54.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:56:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_1.status 2026-02-20T21:56:54.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:56:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_1.status 2026-02-20T21:56:54.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_1.status 2026-02-20T21:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_1.status 2026-02-20T21:56:54.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:56:54.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.status 2026-02-20T21:56:54.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_1 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_1.status 2026-02-20T21:56:54.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:54.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:54.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:54.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_1.status 2026-02-20T21:56:54.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:56:54.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_1 --format xml 2026-02-20T21:56:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:56:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=86 2026-02-20T21:56:54.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:56:54.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:56:54.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:56:54.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.status 2026-02-20T21:56:54.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:54.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:54.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:54.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.status 2026-02-20T21:56:54.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-20T21:56:54.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:56:54.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:56:54.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=35 2026-02-20T21:56:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 86 = 35 2026-02-20T21:56:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:56:54.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:56:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.status 2026-02-20T21:56:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.status 2026-02-20T21:56:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-20T21:56:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:56:55.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:56:55.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=35 2026-02-20T21:56:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 86 = 35 2026-02-20T21:56:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:56:55.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:56:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.status 2026-02-20T21:56:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.status 2026-02-20T21:56:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-20T21:56:56.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:56:56.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:56:56.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:56:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:56:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:56:56.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:56:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:56:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 86 = '' 2026-02-20T21:56:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:56:56.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:56:57.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_1 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.status 2026-02-20T21:56:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.status 2026-02-20T21:56:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_1 --format xml 2026-02-20T21:56:57.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:56:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:56:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=86 2026-02-20T21:56:57.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 86 = 86 2026-02-20T21:56:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2026-02-20T21:56:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:56:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:56:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:56:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:56:57.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:56:58.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2026-02-20T21:56:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_1': 2026-02-20T21:56:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:56:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:56:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:56:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0b4bdd908 2026-02-20T21:56:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0b4bdd908 2026-02-20T21:56:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:56:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:56:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:56:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:56:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:02 2026 2026-02-20T21:56:58.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:02 2026 2026-02-20T21:56:58.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:02 2026 2026-02-20T21:56:58.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:56:58.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:56:58.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:56:58.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: db87f39c-9375-4487-aa96-a4de9e1499de 2026-02-20T21:56:58.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:56:58.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:58.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_1 snap 2026-02-20T21:56:58.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:58.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T21:56:58.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:56:58.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:56:58.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_1.export 2026-02-20T21:56:58.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.export 2026-02-20T21:56:58.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_1.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.export 2026-02-20T21:56:58.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_1.export 2026-02-20T21:56:58.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:56:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.export 2026-02-20T21:56:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:56:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_1.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.export 2026-02-20T21:56:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_1.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_1.export 2026-02-20T21:56:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:56:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:56:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_2 2026-02-20T21:56:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_2 snap 2026-02-20T21:56:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:56:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:56:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:56:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:56:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:57:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2026-02-20T21:57:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:01.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:02.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2026-02-20T21:57:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:57:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:57:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:57:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2026-02-20T21:57:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:57:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:57:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:57:02.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:57:02.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:57:02.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:02.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:57:02.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:57:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:57:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2026-02-20T21:57:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:57:02.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:02.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:57:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:57:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:57:02.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:02.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:02.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:02.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2026-02-20T21:57:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-20T21:57:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:57:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:57:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:57:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2026-02-20T21:57:02.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-20T21:57:02.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:02.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_2 2026-02-20T21:57:02.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:02.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:02.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:57:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:57:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:57:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_2 2026-02-20T21:57:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_2'] 2026-02-20T21:57:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_2 2026-02-20T21:57:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:57:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:57:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:57:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:57:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:57:02.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:02.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:02.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:02.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_2 2026-02-20T21:57:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:57:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:57:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:57:02.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:57:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:57:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_2 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:57:02.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_2 2026-02-20T21:57:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:02.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_2 2026-02-20T21:57:03.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 88 2026-02-20T21:57:03.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_2 2026-02-20T21:57:03.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:57:03.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_2.status 2026-02-20T21:57:03.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:03.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_2.status 2026-02-20T21:57:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_2.status 2026-02-20T21:57:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_2.status 2026-02-20T21:57:03.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.status 2026-02-20T21:57:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_2 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_2.status 2026-02-20T21:57:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_2.status 2026-02-20T21:57:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_2 --format xml 2026-02-20T21:57:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:57:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=88 2026-02-20T21:57:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:03.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:57:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.status 2026-02-20T21:57:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.status 2026-02-20T21:57:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-20T21:57:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:03.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:03.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:57:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:57:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:57:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:57:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 88 = '' 2026-02-20T21:57:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:03.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:57:04.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.status 2026-02-20T21:57:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.status 2026-02-20T21:57:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-20T21:57:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:04.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:57:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:57:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:57:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:57:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 88 = '' 2026-02-20T21:57:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:57:04.666 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:57:04.661+0000 7fa5e2658640 -1 reset not still connected to 0x55b901b836c0 2026-02-20T21:57:04.666 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:57:04.661+0000 7fa5e2658640 -1 reset not still connected to 0x55b9033fc750 2026-02-20T21:57:04.666 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:57:04.661+0000 7fa5e2658640 -1 reset not still connected to 0x55b9052d1e10 2026-02-20T21:57:04.666 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:57:04.661+0000 7fa5e2658640 -1 reset not still connected to 0x55b90790d6c0 2026-02-20T21:57:04.666 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:57:04.661+0000 7fa5e2658640 -1 reset not still connected to 0x55b90790da00 2026-02-20T21:57:04.666 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:57:04.661+0000 7fa5e2658640 -1 reset not still connected to 0x55b91ce06a90 2026-02-20T21:57:04.667 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T21:57:04.661+0000 7fa5e2658640 -1 reset not still connected to 0x55b92c1084e0 2026-02-20T21:57:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_2 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.status 2026-02-20T21:57:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.status 2026-02-20T21:57:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_2 --format xml 2026-02-20T21:57:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:05.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:05.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=88 2026-02-20T21:57:05.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 88 = 88 2026-02-20T21:57:05.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:05.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2026-02-20T21:57:05.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:05.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:05.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:05.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:05.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:05.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:57:05.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2026-02-20T21:57:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_2': 2026-02-20T21:57:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:57:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:57:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:57:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0890d81a7 2026-02-20T21:57:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0890d81a7 2026-02-20T21:57:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:57:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:57:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:57:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:57:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:17 2026 2026-02-20T21:57:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:17 2026 2026-02-20T21:57:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:17 2026 2026-02-20T21:57:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:57:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:57:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:57:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 6f61a2e4-1504-47d7-888d-5162115d91ce 2026-02-20T21:57:06.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:57:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_2 snap 2026-02-20T21:57:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T21:57:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:57:06.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_2.export 2026-02-20T21:57:06.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.export 2026-02-20T21:57:06.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_2.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.export 2026-02-20T21:57:06.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_2.export 2026-02-20T21:57:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:06.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.export 2026-02-20T21:57:06.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:06.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_2.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.export 2026-02-20T21:57:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_2.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_2.export 2026-02-20T21:57:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:57:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_3 2026-02-20T21:57:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_3 snap 2026-02-20T21:57:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:06.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:57:08.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2026-02-20T21:57:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2026-02-20T21:57:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:57:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2026-02-20T21:57:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:57:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:57:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:57:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2026-02-20T21:57:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:57:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:57:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:57:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:57:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:57:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:09.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:57:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:57:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:57:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2026-02-20T21:57:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:57:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:57:09.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:09.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:57:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:57:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2026-02-20T21:57:09.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2026-02-20T21:57:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:57:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:57:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2026-02-20T21:57:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:57:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:09.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_3 2026-02-20T21:57:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:57:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:57:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:09.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:57:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:57:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:57:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_3 2026-02-20T21:57:09.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:57:09.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:09.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_3 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:57:09.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:57:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:57:09.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:09.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:09.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:09.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_3 2026-02-20T21:57:10.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:57:10.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:57:10.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:57:10.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:57:10.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:57:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:57:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:57:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_3 2026-02-20T21:57:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:57:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:57:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_3 2026-02-20T21:57:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_3 2026-02-20T21:57:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 90 2026-02-20T21:57:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_3 2026-02-20T21:57:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:10.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:57:10.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_3.status 2026-02-20T21:57:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:10.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_3.status 2026-02-20T21:57:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_3.status 2026-02-20T21:57:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_3.status 2026-02-20T21:57:10.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:10.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.status 2026-02-20T21:57:10.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_3 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_3.status 2026-02-20T21:57:10.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:10.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:10.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:10.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_3.status 2026-02-20T21:57:10.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_3 --format xml 2026-02-20T21:57:10.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:57:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=90 2026-02-20T21:57:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:57:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.status 2026-02-20T21:57:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.status 2026-02-20T21:57:10.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-20T21:57:10.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=37 2026-02-20T21:57:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 90 = 37 2026-02-20T21:57:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:57:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.status 2026-02-20T21:57:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:11.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.status 2026-02-20T21:57:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-20T21:57:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:11.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=37 2026-02-20T21:57:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 90 = 37 2026-02-20T21:57:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:57:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.status 2026-02-20T21:57:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.status 2026-02-20T21:57:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-20T21:57:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:57:12.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:57:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:57:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:57:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 90 = '' 2026-02-20T21:57:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:12.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:57:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_3 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.status 2026-02-20T21:57:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.status 2026-02-20T21:57:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_3 --format xml 2026-02-20T21:57:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:13.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=90 2026-02-20T21:57:13.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 90 = 90 2026-02-20T21:57:13.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2026-02-20T21:57:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:57:13.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2026-02-20T21:57:14.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_3': 2026-02-20T21:57:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:57:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:57:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:57:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b028f321ba 2026-02-20T21:57:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b028f321ba 2026-02-20T21:57:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:57:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:57:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:57:14.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:57:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:24 2026 2026-02-20T21:57:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:24 2026 2026-02-20T21:57:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:24 2026 2026-02-20T21:57:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:57:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:57:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:57:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: ad6926aa-1d75-40e4-80e4-b13fcd076244 2026-02-20T21:57:14.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:57:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_3 snap 2026-02-20T21:57:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T21:57:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:57:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_3.export 2026-02-20T21:57:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.export 2026-02-20T21:57:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_3.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.export 2026-02-20T21:57:14.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_3.export 2026-02-20T21:57:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:15.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.export 2026-02-20T21:57:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_3.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.export 2026-02-20T21:57:15.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_3.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_3.export 2026-02-20T21:57:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:57:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_4 2026-02-20T21:57:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_4 snap 2026-02-20T21:57:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:15.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:15.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:16.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2026-02-20T21:57:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:57:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:57:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:57:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:57:17.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:57:17.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:17.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:57:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:57:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:57:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-02-20T21:57:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_4'] 2026-02-20T21:57:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2026-02-20T21:57:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:57:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:57:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:57:17.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:57:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:57:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:57:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:17.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2026-02-20T21:57:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_4'] 2026-02-20T21:57:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_4 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:57:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:57:17.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_4 2026-02-20T21:57:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_4'] 2026-02-20T21:57:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_4 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:57:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:57:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:17.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_4 2026-02-20T21:57:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:57:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:57:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:57:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:57:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_4 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_4 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_4 2026-02-20T21:57:18.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 92 2026-02-20T21:57:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_4 2026-02-20T21:57:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:18.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_4.status 2026-02-20T21:57:18.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_4.status 2026-02-20T21:57:18.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_4.status 2026-02-20T21:57:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_4.status 2026-02-20T21:57:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_4.status 2026-02-20T21:57:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.status 2026-02-20T21:57:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_4 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_4.status 2026-02-20T21:57:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_4.status 2026-02-20T21:57:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_4 --format xml 2026-02-20T21:57:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:57:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=92 2026-02-20T21:57:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:57:18.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.status 2026-02-20T21:57:18.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:18.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:18.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:18.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.status 2026-02-20T21:57:18.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-20T21:57:18.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=38 2026-02-20T21:57:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 92 = 38 2026-02-20T21:57:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:18.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:57:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.status 2026-02-20T21:57:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.status 2026-02-20T21:57:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-20T21:57:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=38 2026-02-20T21:57:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 92 = 38 2026-02-20T21:57:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:19.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:57:20.030 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8660974155112928) > weight (0.25) 2026-02-20T21:57:20.030 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6773644054326131) > weight (0.25) 2026-02-20T21:57:20.030 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7901059331713363) > weight (0.25) 2026-02-20T21:57:20.030 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6572744882074159) > weight (0.25) 2026-02-20T21:57:20.031 INFO:tasks.rbd_mirror_thrash:waiting for 85.08379485196376 secs before thrashing 2026-02-20T21:57:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.status 2026-02-20T21:57:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.status 2026-02-20T21:57:20.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-20T21:57:20.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:20.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:20.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:57:20.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:57:20.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:57:20.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:57:20.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 92 = '' 2026-02-20T21:57:20.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:20.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:57:20.560 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:57:20.542+0000 7f1b44dff640 -1 reset not still connected to 0x55f2f10b4340 2026-02-20T21:57:20.560 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:57:20.542+0000 7f1b44dff640 -1 reset not still connected to 0x55f2f10b44e0 2026-02-20T21:57:20.561 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:57:20.542+0000 7f1b44dff640 -1 reset not still connected to 0x55f2f10b5520 2026-02-20T21:57:20.561 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:57:20.542+0000 7f1b44dff640 -1 reset not still connected to 0x55f2f414a8f0 2026-02-20T21:57:20.561 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:57:20.542+0000 7f1b44dff640 -1 reset not still connected to 0x55f2f414bc70 2026-02-20T21:57:20.561 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:57:20.542+0000 7f1b44dff640 -1 reset not still connected to 0x55f2f460c000 2026-02-20T21:57:20.561 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:57:20.542+0000 7f1b44dff640 -1 reset not still connected to 0x55f2f460dd40 2026-02-20T21:57:20.561 INFO:tasks.ceph.cluster1.osd.1.trial032.stderr:2026-02-20T21:57:20.542+0000 7f1b44dff640 -1 reset not still connected to 0x55f2f48d1d40 2026-02-20T21:57:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_4 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.status 2026-02-20T21:57:21.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.status 2026-02-20T21:57:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_4 --format xml 2026-02-20T21:57:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:21.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=92 2026-02-20T21:57:21.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 92 = 92 2026-02-20T21:57:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2026-02-20T21:57:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:57:21.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2026-02-20T21:57:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_4': 2026-02-20T21:57:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:57:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:57:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:57:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b08303fe90 2026-02-20T21:57:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b08303fe90 2026-02-20T21:57:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:34 2026 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:34 2026 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:34 2026 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 60748c73-296b-4b03-b97e-bb38a2fa0c0a 2026-02-20T21:57:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:57:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_4 snap 2026-02-20T21:57:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T21:57:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:57:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_4.export 2026-02-20T21:57:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.export 2026-02-20T21:57:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_4.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.export 2026-02-20T21:57:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_4.export 2026-02-20T21:57:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.export 2026-02-20T21:57:23.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:23.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_4.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.export 2026-02-20T21:57:23.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_4.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_4.export 2026-02-20T21:57:23.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:23.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:57:23.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_5 2026-02-20T21:57:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_5 snap 2026-02-20T21:57:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:57:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2026-02-20T21:57:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2026-02-20T21:57:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:57:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:25.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2026-02-20T21:57:25.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:25.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:25.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:25.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:57:25.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:57:25.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:57:25.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:57:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:57:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2026-02-20T21:57:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:57:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:57:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:57:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:57:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:57:25.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:25.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2026-02-20T21:57:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:57:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_5 2026-02-20T21:57:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:57:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:57:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:57:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:25.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_5 2026-02-20T21:57:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:57:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:25.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_5 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:57:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:57:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:57:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:57:25.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:25.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_5 2026-02-20T21:57:25.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:57:25.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:25.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_5 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:25.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:57:25.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:57:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:57:25.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:57:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_5 2026-02-20T21:57:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:57:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:57:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:57:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:57:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_5 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:57:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:57:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_5 2026-02-20T21:57:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_5 2026-02-20T21:57:26.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 94 2026-02-20T21:57:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_5 2026-02-20T21:57:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:26.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:26.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:57:26.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_5.status 2026-02-20T21:57:26.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_5.status 2026-02-20T21:57:26.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_5.status 2026-02-20T21:57:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_5.status 2026-02-20T21:57:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.status 2026-02-20T21:57:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_5 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_5.status 2026-02-20T21:57:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_5.status 2026-02-20T21:57:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_5 --format xml 2026-02-20T21:57:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:26.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:57:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=94 2026-02-20T21:57:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:57:26.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.status 2026-02-20T21:57:26.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:26.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:26.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:26.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.status 2026-02-20T21:57:26.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-20T21:57:26.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=39 2026-02-20T21:57:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 94 = 39 2026-02-20T21:57:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:57:27.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.status 2026-02-20T21:57:27.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:27.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:27.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:27.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.status 2026-02-20T21:57:27.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-20T21:57:27.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=39 2026-02-20T21:57:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 94 = 39 2026-02-20T21:57:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:57:28.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.status 2026-02-20T21:57:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.status 2026-02-20T21:57:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-20T21:57:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:57:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:57:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 94 = '' 2026-02-20T21:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:57:29.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_5 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.status 2026-02-20T21:57:29.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:29.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:29.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:29.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.status 2026-02-20T21:57:29.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_5 --format xml 2026-02-20T21:57:29.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:29.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=94 2026-02-20T21:57:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 94 = 94 2026-02-20T21:57:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2026-02-20T21:57:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:57:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:30.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2026-02-20T21:57:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_5': 2026-02-20T21:57:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:57:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:57:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:57:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0d9874d2c 2026-02-20T21:57:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0d9874d2c 2026-02-20T21:57:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:57:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:40 2026 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:40 2026 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:40 2026 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 03eab890-30a7-45f9-952e-050bfe363306 2026-02-20T21:57:30.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:57:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_5 snap 2026-02-20T21:57:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T21:57:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:57:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_5.export 2026-02-20T21:57:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.export 2026-02-20T21:57:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_5.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.export 2026-02-20T21:57:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_5.export 2026-02-20T21:57:31.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:31.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.export 2026-02-20T21:57:31.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:31.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_5.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.export 2026-02-20T21:57:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_5.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_5.export 2026-02-20T21:57:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:57:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_6 2026-02-20T21:57:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_6 snap 2026-02-20T21:57:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:57:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2026-02-20T21:57:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:32.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2026-02-20T21:57:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:57:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:32.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2026-02-20T21:57:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:57:33.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:57:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:57:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:33.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:57:33.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:57:33.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:57:33.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:57:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:57:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:57:33.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:33.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:33.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:33.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2026-02-20T21:57:33.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:57:33.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:33.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2026-02-20T21:57:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:57:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:57:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:57:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:57:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:57:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:33.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2026-02-20T21:57:33.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:57:33.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:33.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_6 2026-02-20T21:57:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:57:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:57:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:57:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:57:33.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:57:33.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_6 2026-02-20T21:57:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:57:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:33.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_6 2026-02-20T21:57:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:57:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:33.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:57:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:57:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:57:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:33.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_6 2026-02-20T21:57:34.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:57:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:57:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:57:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:57:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_6 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:57:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:57:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_6 2026-02-20T21:57:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_6 2026-02-20T21:57:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 96 2026-02-20T21:57:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_6 2026-02-20T21:57:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:57:34.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_6.status 2026-02-20T21:57:34.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:34.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_6.status 2026-02-20T21:57:34.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_6.status 2026-02-20T21:57:34.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_6.status 2026-02-20T21:57:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.status 2026-02-20T21:57:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_6 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_6.status 2026-02-20T21:57:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_6.status 2026-02-20T21:57:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_6 --format xml 2026-02-20T21:57:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:57:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=96 2026-02-20T21:57:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:57:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.status 2026-02-20T21:57:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.status 2026-02-20T21:57:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-20T21:57:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:35.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=40 2026-02-20T21:57:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 96 = 40 2026-02-20T21:57:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:35.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:57:35.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.status 2026-02-20T21:57:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.status 2026-02-20T21:57:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-20T21:57:35.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=40 2026-02-20T21:57:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 96 = 40 2026-02-20T21:57:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:35.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:57:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.status 2026-02-20T21:57:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.status 2026-02-20T21:57:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-20T21:57:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:36.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:57:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:57:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:57:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:57:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 96 = '' 2026-02-20T21:57:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:57:37.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_6 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.status 2026-02-20T21:57:37.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:37.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:37.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.status 2026-02-20T21:57:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_6 --format xml 2026-02-20T21:57:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:37.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=96 2026-02-20T21:57:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 96 = 96 2026-02-20T21:57:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2026-02-20T21:57:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:57:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_6': 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b06b510be1 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b06b510be1 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:48:52 2026 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:48:52 2026 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:48:52 2026 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 73eed487-81fa-4ccf-b6ef-2a93a56ed579 2026-02-20T21:57:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_6 snap 2026-02-20T21:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T21:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:38.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:57:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_6.export 2026-02-20T21:57:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.export 2026-02-20T21:57:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_6.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.export 2026-02-20T21:57:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_6.export 2026-02-20T21:57:39.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.export 2026-02-20T21:57:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_6.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.export 2026-02-20T21:57:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_6.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_6.export 2026-02-20T21:57:39.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:39.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:57:39.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_7 2026-02-20T21:57:39.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_7 snap 2026-02-20T21:57:39.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:39.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:39.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:39.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:39.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:57:40.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2026-02-20T21:57:40.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2026-02-20T21:57:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:57:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:57:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:57:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2026-02-20T21:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:57:41.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:57:41.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:57:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:57:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:57:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:57:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:41.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2026-02-20T21:57:41.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:57:41.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:41.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7 2026-02-20T21:57:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:57:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:41.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:41.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:41.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:41.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:41.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:41.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:57:41.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:57:41.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:57:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:41.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7 2026-02-20T21:57:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:57:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:41.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_7 2026-02-20T21:57:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:57:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:57:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:57:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:41.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:57:41.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:57:41.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:41.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:41.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:41.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_7 2026-02-20T21:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_7 2026-02-20T21:57:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:57:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:57:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:57:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:57:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:57:41.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:41.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:41.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:41.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_7 2026-02-20T21:57:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:57:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:57:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:57:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:57:42.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:57:42.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_7 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:57:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:57:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_7 2026-02-20T21:57:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_7 2026-02-20T21:57:42.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 98 2026-02-20T21:57:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_7 2026-02-20T21:57:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:42.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:57:42.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_7.status 2026-02-20T21:57:42.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_7.status 2026-02-20T21:57:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_7.status 2026-02-20T21:57:42.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_7.status 2026-02-20T21:57:42.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.status 2026-02-20T21:57:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_7 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_7.status 2026-02-20T21:57:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_7.status 2026-02-20T21:57:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_7 --format xml 2026-02-20T21:57:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:57:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=98 2026-02-20T21:57:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:57:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.status 2026-02-20T21:57:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:42.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.status 2026-02-20T21:57:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-20T21:57:42.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:43.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=41 2026-02-20T21:57:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 98 = 41 2026-02-20T21:57:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:43.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:57:43.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.status 2026-02-20T21:57:43.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:43.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:43.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:43.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.status 2026-02-20T21:57:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-20T21:57:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=41 2026-02-20T21:57:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 98 = 41 2026-02-20T21:57:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:57:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.status 2026-02-20T21:57:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:44.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.status 2026-02-20T21:57:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-20T21:57:44.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:44.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:44.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:57:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:57:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:57:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:57:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 98 = '' 2026-02-20T21:57:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:57:45.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_7 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.status 2026-02-20T21:57:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.status 2026-02-20T21:57:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_7 --format xml 2026-02-20T21:57:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=98 2026-02-20T21:57:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 98 = 98 2026-02-20T21:57:45.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2026-02-20T21:57:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:57:45.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:46.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2026-02-20T21:57:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_7': 2026-02-20T21:57:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:57:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:57:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0636871b0 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0636871b0 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:07 2026 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:07 2026 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:07 2026 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:57:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:57:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:57:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: de4e320f-58cf-4d0e-b463-cea2ccd8e6c3 2026-02-20T21:57:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:57:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_7 snap 2026-02-20T21:57:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T21:57:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:57:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_7.export 2026-02-20T21:57:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.export 2026-02-20T21:57:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_7.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.export 2026-02-20T21:57:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_7.export 2026-02-20T21:57:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.export 2026-02-20T21:57:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_7.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.export 2026-02-20T21:57:47.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_7.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_7.export 2026-02-20T21:57:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:57:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_8 2026-02-20T21:57:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_8 snap 2026-02-20T21:57:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:57:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2026-02-20T21:57:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2026-02-20T21:57:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:57:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2026-02-20T21:57:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:49.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:57:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:57:49.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:57:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2026-02-20T21:57:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:57:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:57:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:57:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:57:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:57:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:49.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:57:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:57:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:57:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2026-02-20T21:57:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:57:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:57:49.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:57:49.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:57:49.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:57:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:57:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:49.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2026-02-20T21:57:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:57:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_8 2026-02-20T21:57:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:57:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:57:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:57:49.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:57:49.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:57:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_8 2026-02-20T21:57:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:57:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:49.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_8 2026-02-20T21:57:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:57:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:57:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:57:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:57:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_8 2026-02-20T21:57:49.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:57:49.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:49.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_8 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:57:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:57:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:57:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:57:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:49.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:49.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_8 2026-02-20T21:57:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:57:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:57:50.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:57:50.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:57:50.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:57:50.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:57:50.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:57:50.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:50.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_8 2026-02-20T21:57:50.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_8 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:50.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_8 2026-02-20T21:57:50.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 100 2026-02-20T21:57:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_8 2026-02-20T21:57:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:57:50.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_8.status 2026-02-20T21:57:50.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:50.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_8.status 2026-02-20T21:57:50.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_8.status 2026-02-20T21:57:50.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_8.status 2026-02-20T21:57:50.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:50.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.status 2026-02-20T21:57:50.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_8 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_8.status 2026-02-20T21:57:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_8.status 2026-02-20T21:57:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_8 --format xml 2026-02-20T21:57:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:50.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:57:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=100 2026-02-20T21:57:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:57:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.status 2026-02-20T21:57:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.status 2026-02-20T21:57:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-20T21:57:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=42 2026-02-20T21:57:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 100 = 42 2026-02-20T21:57:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:57:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.status 2026-02-20T21:57:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.status 2026-02-20T21:57:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-20T21:57:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:51.461 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:51.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=42 2026-02-20T21:57:51.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 100 = 42 2026-02-20T21:57:51.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:51.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:57:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.status 2026-02-20T21:57:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.status 2026-02-20T21:57:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-20T21:57:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:52.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:52.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:57:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:57:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:57:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:57:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 100 = '' 2026-02-20T21:57:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:57:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_8 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.status 2026-02-20T21:57:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.status 2026-02-20T21:57:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_8 --format xml 2026-02-20T21:57:53.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=100 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 100 = 100 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:57:53.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:54.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2026-02-20T21:57:54.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_8': 2026-02-20T21:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0aed5fa92 2026-02-20T21:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0aed5fa92 2026-02-20T21:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:57:54.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:57:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:17 2026 2026-02-20T21:57:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:17 2026 2026-02-20T21:57:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:17 2026 2026-02-20T21:57:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:57:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:57:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:57:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: b550a116-b65a-457c-854a-75ae61a3215e 2026-02-20T21:57:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:57:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_8 snap 2026-02-20T21:57:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T21:57:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:57:54.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_8.export 2026-02-20T21:57:54.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.export 2026-02-20T21:57:54.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_8.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.export 2026-02-20T21:57:54.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_8.export 2026-02-20T21:57:55.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.export 2026-02-20T21:57:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:57:55.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_8.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.export 2026-02-20T21:57:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_8.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_8.export 2026-02-20T21:57:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:57:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_9 2026-02-20T21:57:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_9 snap 2026-02-20T21:57:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:57:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:57:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2026-02-20T21:57:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:56.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2026-02-20T21:57:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:57:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:57:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:56.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:57:57.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2026-02-20T21:57:57.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:57.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:57.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:57.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:57:57.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:57:57.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:57:57.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2026-02-20T21:57:57.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:57:57.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:57:57.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:57:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:57:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:57:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:57:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:57:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:57:57.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2026-02-20T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:57:57.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:57.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:57:57.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:57:57.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:57.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:57.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:57:57.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2026-02-20T21:57:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:57:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:57.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:57.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2026-02-20T21:57:57.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:57:57.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:57.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:57.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:57.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:57:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:57:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:57:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:57:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:57:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:57:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2026-02-20T21:57:57.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:57:57.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:57.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_9 2026-02-20T21:57:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:57:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:57:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:57:57.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:57:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:57:57.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:57:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:57:57.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_9 2026-02-20T21:57:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:57:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:57:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_9 2026-02-20T21:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:57:57.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:57:57.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:57:57.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:57:57.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:57:57.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:57:57.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:57:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:57:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_9 2026-02-20T21:57:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:57:58.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:57:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:57:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:57:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:57:58.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_9 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:57:58.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_9 2026-02-20T21:57:58.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:58.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:58.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:58.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_9 2026-02-20T21:57:58.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 102 2026-02-20T21:57:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_9 2026-02-20T21:57:58.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:57:58.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:58.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:57:58.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:57:58.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:58.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:57:58.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_9.status 2026-02-20T21:57:58.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:58.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_9.status 2026-02-20T21:57:58.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_9.status 2026-02-20T21:57:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_9.status 2026-02-20T21:57:58.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.status 2026-02-20T21:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_9 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_9.status 2026-02-20T21:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_9.status 2026-02-20T21:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_9 --format xml 2026-02-20T21:57:58.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:57:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=102 2026-02-20T21:57:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:57:58.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:58.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:57:59.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.status 2026-02-20T21:57:59.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:59.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:59.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:59.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.status 2026-02-20T21:57:59.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-20T21:57:59.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=43 2026-02-20T21:57:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 102 = 43 2026-02-20T21:57:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:57:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.status 2026-02-20T21:57:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:57:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:57:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:57:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.status 2026-02-20T21:57:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-20T21:57:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:57:59.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:57:59.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=43 2026-02-20T21:57:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 102 = 43 2026-02-20T21:57:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:57:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:58:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.status 2026-02-20T21:58:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:58:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.status 2026-02-20T21:58:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-20T21:58:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:00.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:00.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:00.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 102 = '' 2026-02-20T21:58:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:00.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:58:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_9 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.status 2026-02-20T21:58:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:58:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.status 2026-02-20T21:58:01.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_9 --format xml 2026-02-20T21:58:01.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=102 2026-02-20T21:58:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 102 = 102 2026-02-20T21:58:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2026-02-20T21:58:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:58:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:58:01.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2026-02-20T21:58:02.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_9': 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b08baaae98 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b08baaae98 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:58:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:32 2026 2026-02-20T21:58:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:32 2026 2026-02-20T21:58:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:32 2026 2026-02-20T21:58:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:58:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:58:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:58:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 3db04467-75e8-4151-b6e5-29a86d819303 2026-02-20T21:58:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:58:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_9 snap 2026-02-20T21:58:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T21:58:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:58:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_9.export 2026-02-20T21:58:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.export 2026-02-20T21:58:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_9.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.export 2026-02-20T21:58:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_9.export 2026-02-20T21:58:03.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:03.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.export 2026-02-20T21:58:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:03.319 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_9.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.export 2026-02-20T21:58:03.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_9.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_9.export 2026-02-20T21:58:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:58:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_10 2026-02-20T21:58:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_10 snap 2026-02-20T21:58:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:58:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2026-02-20T21:58:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2026-02-20T21:58:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:04.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:04.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:58:04.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:04.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:04.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2026-02-20T21:58:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:58:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:58:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:58:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2026-02-20T21:58:05.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:58:05.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:58:05.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:05.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:58:05.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:58:05.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:58:05.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:05.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:58:05.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:58:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:58:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-02-20T21:58:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:58:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:58:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:05.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:58:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:58:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:58:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:05.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-02-20T21:58:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-20T21:58:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:05.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:05.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:05.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2026-02-20T21:58:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:58:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:58:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:58:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:58:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:58:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:58:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2026-02-20T21:58:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-20T21:58:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:05.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_10 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:58:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:58:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:58:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_10 2026-02-20T21:58:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-20T21:58:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_10 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:58:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:58:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:58:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:58:05.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:05.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:05.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_10 2026-02-20T21:58:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:58:06.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:58:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:58:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:58:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:58:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:58:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:58:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_10 2026-02-20T21:58:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_10 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_10 2026-02-20T21:58:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 104 2026-02-20T21:58:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_10 2026-02-20T21:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_10.status 2026-02-20T21:58:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_10.status 2026-02-20T21:58:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_10.status 2026-02-20T21:58:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_10.status 2026-02-20T21:58:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_10.status 2026-02-20T21:58:06.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:06.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.status 2026-02-20T21:58:06.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_10 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_10.status 2026-02-20T21:58:06.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:06.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_10.status 2026-02-20T21:58:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_10 --format xml 2026-02-20T21:58:06.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:58:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=104 2026-02-20T21:58:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:58:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.status 2026-02-20T21:58:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.status 2026-02-20T21:58:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-20T21:58:07.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=44 2026-02-20T21:58:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 104 = 44 2026-02-20T21:58:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:07.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:58:07.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.status 2026-02-20T21:58:07.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.status 2026-02-20T21:58:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-20T21:58:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:07.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=44 2026-02-20T21:58:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 104 = 44 2026-02-20T21:58:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:58:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.status 2026-02-20T21:58:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.status 2026-02-20T21:58:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-20T21:58:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:08.325 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 104 = '' 2026-02-20T21:58:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:58:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_10 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.status 2026-02-20T21:58:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:09.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.status 2026-02-20T21:58:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_10 --format xml 2026-02-20T21:58:09.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:09.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=104 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 104 = 104 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:58:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:10.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2026-02-20T21:58:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_10': 2026-02-20T21:58:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:58:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:58:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:58:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b08b3dc56a 2026-02-20T21:58:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b08b3dc56a 2026-02-20T21:58:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:58:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:58:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:58:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:58:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:38 2026 2026-02-20T21:58:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:38 2026 2026-02-20T21:58:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:38 2026 2026-02-20T21:58:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:58:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:58:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:58:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 73f7f3f5-fe4b-491f-8f59-209e2587d369 2026-02-20T21:58:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:58:10.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:10.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_10 snap 2026-02-20T21:58:10.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:10.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T21:58:10.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:10.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:58:10.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_10.export 2026-02-20T21:58:10.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.export 2026-02-20T21:58:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_10.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.export 2026-02-20T21:58:10.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_10.export 2026-02-20T21:58:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:11.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.export 2026-02-20T21:58:11.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:11.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_10.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.export 2026-02-20T21:58:11.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_10.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_10.export 2026-02-20T21:58:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:58:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_11 2026-02-20T21:58:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_11 snap 2026-02-20T21:58:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:11.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:11.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:11.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:11.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:58:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2026-02-20T21:58:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:12.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2026-02-20T21:58:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:58:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:12.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:13.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2026-02-20T21:58:13.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:13.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:13.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:13.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:58:13.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:58:13.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:58:13.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2026-02-20T21:58:13.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:58:13.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:58:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:58:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:58:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:58:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:13.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:58:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:58:13.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:13.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:58:13.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:58:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:58:13.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:58:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-20T21:58:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:58:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-20T21:58:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:58:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:58:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:58:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:58:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:58:13.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:13.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:13.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:13.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-20T21:58:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:58:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:13.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:58:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:58:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:58:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:58:13.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:13.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-20T21:58:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:58:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2026-02-20T21:58:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:58:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:14.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:14.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:58:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:58:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:58:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:14.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:14.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:14.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2026-02-20T21:58:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:58:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:58:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:58:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:58:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:58:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:58:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:58:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_11 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_11 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:14.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:14.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_11 2026-02-20T21:58:14.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 106 2026-02-20T21:58:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_11 2026-02-20T21:58:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:58:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_11.status 2026-02-20T21:58:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_11.status 2026-02-20T21:58:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_11.status 2026-02-20T21:58:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_11.status 2026-02-20T21:58:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.status 2026-02-20T21:58:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_11 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_11.status 2026-02-20T21:58:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_11.status 2026-02-20T21:58:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_11 --format xml 2026-02-20T21:58:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:14.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=106 2026-02-20T21:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:14.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:58:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.status 2026-02-20T21:58:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.status 2026-02-20T21:58:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-20T21:58:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:15.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=45 2026-02-20T21:58:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 106 = 45 2026-02-20T21:58:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:15.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:58:15.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.status 2026-02-20T21:58:15.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.status 2026-02-20T21:58:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-20T21:58:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:15.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=45 2026-02-20T21:58:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 106 = 45 2026-02-20T21:58:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:15.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:58:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.status 2026-02-20T21:58:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.status 2026-02-20T21:58:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-20T21:58:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 106 = '' 2026-02-20T21:58:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:58:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_11 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.status 2026-02-20T21:58:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.status 2026-02-20T21:58:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_11 --format xml 2026-02-20T21:58:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=106 2026-02-20T21:58:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 106 = 106 2026-02-20T21:58:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2026-02-20T21:58:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:18.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:18.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:18.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:58:18.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:19.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2026-02-20T21:58:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_11': 2026-02-20T21:58:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:58:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:58:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0da426cf2 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0da426cf2 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:46 2026 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:46 2026 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:46 2026 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:58:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:58:19.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:58:19.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: bb0f4c60-0c8c-41c3-9fc2-067109d74988 2026-02-20T21:58:19.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:58:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_11 snap 2026-02-20T21:58:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T21:58:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:58:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_11.export 2026-02-20T21:58:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.export 2026-02-20T21:58:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_11.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.export 2026-02-20T21:58:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_11.export 2026-02-20T21:58:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:19.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.export 2026-02-20T21:58:19.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_11.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.export 2026-02-20T21:58:19.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_11.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_11.export 2026-02-20T21:58:19.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:19.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:58:19.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_12 2026-02-20T21:58:19.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_12 snap 2026-02-20T21:58:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:19.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:58:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2026-02-20T21:58:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:21.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2026-02-20T21:58:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:58:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:58:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:58:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:58:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-20T21:58:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:58:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:22.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:58:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:58:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:58:22.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:22.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-20T21:58:22.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:58:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:22.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-20T21:58:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:58:22.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:58:22.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:22.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:22.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:22.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:58:22.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:58:22.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:58:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-20T21:58:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:58:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2026-02-20T21:58:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:22.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:22.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:22.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:22.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:22.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:58:22.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:58:22.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:58:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:22.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2026-02-20T21:58:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:58:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_12 2026-02-20T21:58:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:58:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:58:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:58:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:58:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:58:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_12 2026-02-20T21:58:22.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:58:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:58:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:58:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:58:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_12 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:58:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:58:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_12 2026-02-20T21:58:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_12 2026-02-20T21:58:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 108 2026-02-20T21:58:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_12 2026-02-20T21:58:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:23.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:58:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_12.status 2026-02-20T21:58:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_12.status 2026-02-20T21:58:23.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_12.status 2026-02-20T21:58:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_12.status 2026-02-20T21:58:23.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:23.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.status 2026-02-20T21:58:23.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_12 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_12.status 2026-02-20T21:58:23.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:23.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:23.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:23.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_12.status 2026-02-20T21:58:23.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_12 --format xml 2026-02-20T21:58:23.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:58:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=108 2026-02-20T21:58:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:58:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.status 2026-02-20T21:58:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.status 2026-02-20T21:58:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-20T21:58:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 108 = '' 2026-02-20T21:58:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:58:24.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.status 2026-02-20T21:58:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.status 2026-02-20T21:58:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-20T21:58:24.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:24.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:24.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:24.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:24.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:24.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:24.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:24.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 108 = '' 2026-02-20T21:58:24.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:24.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:58:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_12 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.status 2026-02-20T21:58:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.status 2026-02-20T21:58:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_12 --format xml 2026-02-20T21:58:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=108 2026-02-20T21:58:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 108 = 108 2026-02-20T21:58:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2026-02-20T21:58:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:58:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2026-02-20T21:58:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_12': 2026-02-20T21:58:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:58:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:58:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:58:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b057f5b276 2026-02-20T21:58:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b057f5b276 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:49:52 2026 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:49:52 2026 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:49:52 2026 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:58:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: ffaa636b-9c5c-4539-965b-b5870a492843 2026-02-20T21:58:26.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:58:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_12 snap 2026-02-20T21:58:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T21:58:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:58:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_12.export 2026-02-20T21:58:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.export 2026-02-20T21:58:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_12.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.export 2026-02-20T21:58:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_12.export 2026-02-20T21:58:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:26.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.export 2026-02-20T21:58:26.414 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.81268225641623) > weight (0.25) 2026-02-20T21:58:26.414 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4457194771886497) > weight (0.25) 2026-02-20T21:58:26.414 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9770917610560552) > weight (0.25) 2026-02-20T21:58:26.414 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.5794034075211568) > weight (0.25) 2026-02-20T21:58:26.414 INFO:tasks.rbd_mirror_thrash:waiting for 106.72814530921093 secs before thrashing 2026-02-20T21:58:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:26.462 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_12.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.export 2026-02-20T21:58:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_12.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_12.export 2026-02-20T21:58:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:58:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_13 2026-02-20T21:58:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_13 snap 2026-02-20T21:58:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:26.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:58:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:28.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:29.551 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2026-02-20T21:58:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:58:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:58:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:58:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2026-02-20T21:58:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:58:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:58:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:58:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:58:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:58:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:58:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:58:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:58:29.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:29.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:29.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:29.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-20T21:58:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:58:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:29.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:58:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:58:29.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:29.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-20T21:58:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:58:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_13 2026-02-20T21:58:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:58:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:58:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:58:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:58:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_13 2026-02-20T21:58:29.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:58:29.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:29.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_13 2026-02-20T21:58:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:58:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:58:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:58:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:58:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:58:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:58:29.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:58:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_13 2026-02-20T21:58:29.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:58:29.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:58:29.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:58:29.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:58:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_13 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:58:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:58:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_13 2026-02-20T21:58:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_13 2026-02-20T21:58:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 110 2026-02-20T21:58:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_13 2026-02-20T21:58:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:58:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_13.status 2026-02-20T21:58:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_13.status 2026-02-20T21:58:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_13.status 2026-02-20T21:58:30.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_13.status 2026-02-20T21:58:30.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.status 2026-02-20T21:58:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_13 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_13.status 2026-02-20T21:58:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_13.status 2026-02-20T21:58:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_13 --format xml 2026-02-20T21:58:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:58:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=110 2026-02-20T21:58:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:58:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.status 2026-02-20T21:58:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.status 2026-02-20T21:58:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-20T21:58:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:30.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=47 2026-02-20T21:58:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 110 = 47 2026-02-20T21:58:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:58:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.status 2026-02-20T21:58:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.status 2026-02-20T21:58:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-20T21:58:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:31.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=47 2026-02-20T21:58:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 110 = 47 2026-02-20T21:58:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:31.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:58:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.status 2026-02-20T21:58:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.status 2026-02-20T21:58:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-20T21:58:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:32.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:32.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 110 = '' 2026-02-20T21:58:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:58:33.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_13 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.status 2026-02-20T21:58:33.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.status 2026-02-20T21:58:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_13 --format xml 2026-02-20T21:58:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:33.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=110 2026-02-20T21:58:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 110 = 110 2026-02-20T21:58:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2026-02-20T21:58:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:33.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:58:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:34.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2026-02-20T21:58:34.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_13': 2026-02-20T21:58:34.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:58:34.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:58:34.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:58:34.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b01e366814 2026-02-20T21:58:34.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b01e366814 2026-02-20T21:58:34.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:58:34.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:58:34.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:58:34.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:58:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:01 2026 2026-02-20T21:58:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:01 2026 2026-02-20T21:58:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:01 2026 2026-02-20T21:58:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:58:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:58:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:58:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 9cfb07c7-2733-4bb0-9788-9c939376ecdc 2026-02-20T21:58:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:58:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_13 snap 2026-02-20T21:58:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T21:58:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:58:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_13.export 2026-02-20T21:58:34.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.export 2026-02-20T21:58:34.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_13.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.export 2026-02-20T21:58:34.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_13.export 2026-02-20T21:58:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:34.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.export 2026-02-20T21:58:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_13.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.export 2026-02-20T21:58:35.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_13.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_13.export 2026-02-20T21:58:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:58:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_14 2026-02-20T21:58:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_14 snap 2026-02-20T21:58:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:58:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:36.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2026-02-20T21:58:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:58:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:37.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:58:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:58:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:58:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:58:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:37.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:58:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:58:37.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:58:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:37.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:37.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:37.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2026-02-20T21:58:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:58:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:37.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:58:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:58:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:58:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:37.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:37.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:37.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2026-02-20T21:58:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:58:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2026-02-20T21:58:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:58:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:58:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:58:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:37.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:58:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:58:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:58:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2026-02-20T21:58:37.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:58:37.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:37.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_14 2026-02-20T21:58:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:58:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:58:37.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:58:37.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:58:37.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:58:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_14 2026-02-20T21:58:37.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:58:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:58:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:58:37.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:58:37.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:58:37.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_14 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:58:37.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_14 2026-02-20T21:58:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:37.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_14 2026-02-20T21:58:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 112 2026-02-20T21:58:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_14 2026-02-20T21:58:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_14.status 2026-02-20T21:58:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_14.status 2026-02-20T21:58:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_14.status 2026-02-20T21:58:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_14.status 2026-02-20T21:58:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.status 2026-02-20T21:58:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_14 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_14.status 2026-02-20T21:58:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_14.status 2026-02-20T21:58:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_14 --format xml 2026-02-20T21:58:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:58:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=112 2026-02-20T21:58:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:58:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.status 2026-02-20T21:58:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.status 2026-02-20T21:58:38.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-20T21:58:38.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:38.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:38.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:38.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:38.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:38.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:38.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:38.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 112 = '' 2026-02-20T21:58:38.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:38.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:58:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.status 2026-02-20T21:58:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.status 2026-02-20T21:58:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-20T21:58:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:39.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:39.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:39.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:39.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:39.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 112 = '' 2026-02-20T21:58:39.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:39.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:58:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_14 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.status 2026-02-20T21:58:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.status 2026-02-20T21:58:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_14 --format xml 2026-02-20T21:58:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:40.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=112 2026-02-20T21:58:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 112 = 112 2026-02-20T21:58:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:40.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2026-02-20T21:58:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:58:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:41.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2026-02-20T21:58:41.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_14': 2026-02-20T21:58:41.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b090f299ce 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b090f299ce 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:10 2026 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:10 2026 2026-02-20T21:58:41.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:10 2026 2026-02-20T21:58:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:58:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:58:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:58:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 14383d26-7af3-40c8-90a8-27603ddc790c 2026-02-20T21:58:41.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:58:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_14 snap 2026-02-20T21:58:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T21:58:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:58:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_14.export 2026-02-20T21:58:41.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.export 2026-02-20T21:58:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_14.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.export 2026-02-20T21:58:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_14.export 2026-02-20T21:58:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:41.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.export 2026-02-20T21:58:41.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:41.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_14.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.export 2026-02-20T21:58:41.547 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_14.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_14.export 2026-02-20T21:58:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:58:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_15 2026-02-20T21:58:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_15 snap 2026-02-20T21:58:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:58:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2026-02-20T21:58:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2026-02-20T21:58:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:58:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2026-02-20T21:58:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:58:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:58:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:58:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2026-02-20T21:58:44.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:58:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:58:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:58:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:58:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:58:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:44.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:58:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:58:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:58:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:58:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:58:44.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:44.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:44.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:44.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2026-02-20T21:58:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:58:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:44.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2026-02-20T21:58:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:58:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:58:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:58:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:58:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:44.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:58:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:58:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:58:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2026-02-20T21:58:44.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:58:44.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:44.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_15 2026-02-20T21:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:58:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:58:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:58:44.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:44.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:44.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:44.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_15 2026-02-20T21:58:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:58:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_15 2026-02-20T21:58:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:58:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:58:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_15 2026-02-20T21:58:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:58:44.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:58:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:58:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:58:44.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:58:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:58:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:58:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:44.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_15 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_15 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:44.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:44.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_15 2026-02-20T21:58:45.115 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9902915793798887) > weight (0.25) 2026-02-20T21:58:45.115 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-20T21:58:45.115 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-20T21:58:45.116 INFO:tasks.rbd_mirror_thrash:waiting for 0.11082759537812237 secs before reviving daemons 2026-02-20T21:58:45.116 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T21:58:45.111+0000 7ffab9f6d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 21321) UID: 1000 2026-02-20T21:58:45.227 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-20T21:58:45.228 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:58:45.322 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-20T21:58:45.322 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T21:58:45.322 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T21:58:45.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T21:58:45.370 INFO:tasks.rbd_mirror_thrash:waiting for 88.28081731670514 secs before thrashing 2026-02-20T21:58:45.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 114 2026-02-20T21:58:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_15 2026-02-20T21:58:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:58:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_15.status 2026-02-20T21:58:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_15.status 2026-02-20T21:58:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_15.status 2026-02-20T21:58:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_15.status 2026-02-20T21:58:45.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.status 2026-02-20T21:58:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_15 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_15.status 2026-02-20T21:58:45.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_15.status 2026-02-20T21:58:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_15 --format xml 2026-02-20T21:58:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:45.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:58:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=114 2026-02-20T21:58:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:58:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.status 2026-02-20T21:58:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.status 2026-02-20T21:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-20T21:58:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:45.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 114 = '' 2026-02-20T21:58:45.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:45.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:58:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.status 2026-02-20T21:58:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:46.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.status 2026-02-20T21:58:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-20T21:58:46.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:46.294 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:46.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:46.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 114 = '' 2026-02-20T21:58:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:46.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:58:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_15 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.status 2026-02-20T21:58:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.status 2026-02-20T21:58:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_15 --format xml 2026-02-20T21:58:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:47.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:47.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=114 2026-02-20T21:58:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 114 = 114 2026-02-20T21:58:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2026-02-20T21:58:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:58:47.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:48.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2026-02-20T21:58:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_15': 2026-02-20T21:58:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:58:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:58:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:58:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0296f5eed 2026-02-20T21:58:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0296f5eed 2026-02-20T21:58:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:58:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:58:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:58:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:58:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:22 2026 2026-02-20T21:58:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:22 2026 2026-02-20T21:58:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:22 2026 2026-02-20T21:58:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:58:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:58:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:58:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: d173de17-b1d5-4604-94f9-5797e13957df 2026-02-20T21:58:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:58:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_15 snap 2026-02-20T21:58:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T21:58:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:58:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_15.export 2026-02-20T21:58:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.export 2026-02-20T21:58:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_15.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.export 2026-02-20T21:58:48.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_15.export 2026-02-20T21:58:48.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:48.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.export 2026-02-20T21:58:48.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:48.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_15.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.export 2026-02-20T21:58:48.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_15.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_15.export 2026-02-20T21:58:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:58:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_16 2026-02-20T21:58:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_16 snap 2026-02-20T21:58:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:50.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2026-02-20T21:58:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:58:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:58:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:58:51.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:58:51.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:58:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-20T21:58:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:58:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:58:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:58:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:51.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:58:51.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:58:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:58:51.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:51.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:51.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:51.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-20T21:58:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:58:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-02-20T21:58:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:58:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:58:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:58:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:58:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:51.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:58:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:58:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:58:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:58:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:58:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:58:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:58:51.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:58:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:51.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:51.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-02-20T21:58:51.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:58:51.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:51.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:58:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:58:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2026-02-20T21:58:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:58:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:51.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:51.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2026-02-20T21:58:51.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:58:51.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:51.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:51.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:51.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:51.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:51.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:58:51.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:58:51.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:58:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2026-02-20T21:58:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:58:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:58:51.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:58:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:58:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:58:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:58:51.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_16 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_16 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_16 2026-02-20T21:58:52.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 116 2026-02-20T21:58:52.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_16 2026-02-20T21:58:52.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:52.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:58:52.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_16.status 2026-02-20T21:58:52.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:52.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_16.status 2026-02-20T21:58:52.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_16.status 2026-02-20T21:58:52.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_16.status 2026-02-20T21:58:52.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.status 2026-02-20T21:58:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_16 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_16.status 2026-02-20T21:58:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_16.status 2026-02-20T21:58:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_16 --format xml 2026-02-20T21:58:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:58:52.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=116 2026-02-20T21:58:52.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:52.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:52.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:58:52.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.status 2026-02-20T21:58:52.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:52.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:52.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:52.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.status 2026-02-20T21:58:52.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-20T21:58:52.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:52.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 116 = '' 2026-02-20T21:58:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:58:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.status 2026-02-20T21:58:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.status 2026-02-20T21:58:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-20T21:58:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:53.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:53.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:53.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:58:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:58:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:58:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:58:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 116 = '' 2026-02-20T21:58:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:53.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:58:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_16 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.status 2026-02-20T21:58:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.status 2026-02-20T21:58:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_16 --format xml 2026-02-20T21:58:54.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=116 2026-02-20T21:58:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 116 = 116 2026-02-20T21:58:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2026-02-20T21:58:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:58:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:55.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2026-02-20T21:58:55.374 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_16': 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b099455b13 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b099455b13 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:58:55.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:29 2026 2026-02-20T21:58:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:29 2026 2026-02-20T21:58:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:29 2026 2026-02-20T21:58:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:58:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:58:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:58:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: f3776a65-963f-4287-81b8-3f98573893e1 2026-02-20T21:58:55.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:58:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_16 snap 2026-02-20T21:58:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T21:58:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:58:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_16.export 2026-02-20T21:58:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.export 2026-02-20T21:58:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_16.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.export 2026-02-20T21:58:55.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_16.export 2026-02-20T21:58:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:55.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.export 2026-02-20T21:58:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:58:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_16.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.export 2026-02-20T21:58:55.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_16.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_16.export 2026-02-20T21:58:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:58:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_17 2026-02-20T21:58:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_17 snap 2026-02-20T21:58:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:58:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:58:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:58:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2026-02-20T21:58:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:58:58.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2026-02-20T21:58:58.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:58.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:58.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:58:58.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:58:58.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:58:58.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:58:58.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2026-02-20T21:58:58.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:58:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:58:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:58:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:58:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:58:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:58:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:58:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:58:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:58:58.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:58.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:58.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:58.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:58:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:58:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:58:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:58:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:58:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:58:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2026-02-20T21:58:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-20T21:58:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2026-02-20T21:58:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:58:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:58.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:58:58.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:58.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:58:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:58:58.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:58:58.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:58.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:58.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:58:58.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2026-02-20T21:58:58.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-20T21:58:58.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:58.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:58.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:58.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_17 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:58.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:58.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:58:58.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:58:58.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:58.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:58.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:58:58.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:58:58.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:58:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:58.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:58:58.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_17 2026-02-20T21:58:58.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-20T21:58:58.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:58:58.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:58:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:58:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_17 2026-02-20T21:58:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:58:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:58:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:58:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:58:58.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:58:58.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:58:58.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:58:58.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:58:58.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:58.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:58:58.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_17 2026-02-20T21:58:58.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:58:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:58:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:58:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:58:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:58:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:58:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:58:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:58:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_17 2026-02-20T21:58:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:58:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:58:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:58:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_17 2026-02-20T21:58:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:58.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:58:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_17 2026-02-20T21:58:59.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 118 2026-02-20T21:58:59.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_17 2026-02-20T21:58:59.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:58:59.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:59.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:58:59.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:58:59.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:58:59.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:58:59.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_17.status 2026-02-20T21:58:59.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_17.status 2026-02-20T21:58:59.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_17.status 2026-02-20T21:58:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_17.status 2026-02-20T21:58:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:58:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.status 2026-02-20T21:58:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_17 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_17.status 2026-02-20T21:58:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:58:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:58:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_17.status 2026-02-20T21:58:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_17 --format xml 2026-02-20T21:58:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:59.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:58:59.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=118 2026-02-20T21:58:59.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:58:59.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:59.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:58:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.status 2026-02-20T21:58:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:58:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:58:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:58:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.status 2026-02-20T21:58:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-20T21:58:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:58:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:58:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=51 2026-02-20T21:58:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 118 = 51 2026-02-20T21:58:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:58:59.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:59:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.status 2026-02-20T21:59:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:59:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.status 2026-02-20T21:59:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-20T21:59:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:00.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:00.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=51 2026-02-20T21:59:00.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 118 = 51 2026-02-20T21:59:00.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:00.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:59:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.status 2026-02-20T21:59:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:59:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.status 2026-02-20T21:59:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-20T21:59:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:01.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:01.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:01.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:59:01.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:59:01.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:59:01.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:59:01.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 118 = '' 2026-02-20T21:59:01.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:01.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:59:02.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_17 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.status 2026-02-20T21:59:02.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:02.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:59:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.status 2026-02-20T21:59:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_17 --format xml 2026-02-20T21:59:02.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:02.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=118 2026-02-20T21:59:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 118 = 118 2026-02-20T21:59:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2026-02-20T21:59:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:59:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:59:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:03.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2026-02-20T21:59:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_17': 2026-02-20T21:59:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:59:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:59:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0390e8083 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0390e8083 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:40 2026 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:40 2026 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:40 2026 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:59:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:59:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: e65dbfbb-2857-4359-9849-aa0a0d8627ec 2026-02-20T21:59:03.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:59:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_17 snap 2026-02-20T21:59:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:03.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T21:59:03.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:03.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:59:03.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_17.export 2026-02-20T21:59:03.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.export 2026-02-20T21:59:03.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_17.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.export 2026-02-20T21:59:03.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_17.export 2026-02-20T21:59:03.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.export 2026-02-20T21:59:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:04.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_17.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.export 2026-02-20T21:59:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_17.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_17.export 2026-02-20T21:59:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:59:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_18 2026-02-20T21:59:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_18 snap 2026-02-20T21:59:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:59:05.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2026-02-20T21:59:05.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:05.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:05.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2026-02-20T21:59:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:59:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2026-02-20T21:59:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:59:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:59:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:59:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2026-02-20T21:59:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:59:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:59:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:59:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:59:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:59:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:59:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:59:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:59:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:59:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:59:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2026-02-20T21:59:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:59:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:59:06.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:59:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:59:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18 2026-02-20T21:59:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:59:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_18 2026-02-20T21:59:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:59:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:59:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:59:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:59:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:59:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:59:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_18 2026-02-20T21:59:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:59:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_18 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:59:06.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:06.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:06.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:06.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_18 2026-02-20T21:59:06.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:59:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:59:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:59:06.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:59:06.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:59:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:59:06.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_18 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_18 2026-02-20T21:59:06.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:06.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:06.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:06.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_18 2026-02-20T21:59:07.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 120 2026-02-20T21:59:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_18 2026-02-20T21:59:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:59:07.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_18.status 2026-02-20T21:59:07.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:07.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_18.status 2026-02-20T21:59:07.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_18.status 2026-02-20T21:59:07.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_18.status 2026-02-20T21:59:07.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.status 2026-02-20T21:59:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_18 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_18.status 2026-02-20T21:59:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:07.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:07.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_18.status 2026-02-20T21:59:07.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_18 --format xml 2026-02-20T21:59:07.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:59:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=120 2026-02-20T21:59:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:07.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:59:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.status 2026-02-20T21:59:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.status 2026-02-20T21:59:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-20T21:59:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=52 2026-02-20T21:59:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 120 = 52 2026-02-20T21:59:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:59:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.status 2026-02-20T21:59:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.status 2026-02-20T21:59:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-20T21:59:08.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=52 2026-02-20T21:59:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 120 = 52 2026-02-20T21:59:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:59:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.status 2026-02-20T21:59:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.status 2026-02-20T21:59:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-20T21:59:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:09.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:09.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:09.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:59:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:59:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:59:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:59:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 120 = '' 2026-02-20T21:59:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:09.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:59:10.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_18 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.status 2026-02-20T21:59:10.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:10.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:10.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:10.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.status 2026-02-20T21:59:10.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_18 --format xml 2026-02-20T21:59:10.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:10.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=120 2026-02-20T21:59:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 120 = 120 2026-02-20T21:59:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2026-02-20T21:59:10.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:10.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:10.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:10.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:10.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:10.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:59:10.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:11.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2026-02-20T21:59:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_18': 2026-02-20T21:59:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:59:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:59:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:59:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0f7f836a4 2026-02-20T21:59:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0f7f836a4 2026-02-20T21:59:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:59:11.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:50:51 2026 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:50:51 2026 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:50:51 2026 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: a7e538f7-880a-4d2e-9bda-105c14b4adf7 2026-02-20T21:59:11.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:59:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_18 snap 2026-02-20T21:59:11.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T21:59:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:59:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_18.export 2026-02-20T21:59:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.export 2026-02-20T21:59:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_18.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.export 2026-02-20T21:59:11.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_18.export 2026-02-20T21:59:11.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.export 2026-02-20T21:59:12.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:12.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_18.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.export 2026-02-20T21:59:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_18.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_18.export 2026-02-20T21:59:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:59:12.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_19 2026-02-20T21:59:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_19 snap 2026-02-20T21:59:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:12.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:59:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2026-02-20T21:59:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2026-02-20T21:59:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:13.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:59:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:13.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2026-02-20T21:59:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:59:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:59:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:59:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2026-02-20T21:59:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:59:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:59:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:59:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:59:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:59:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:59:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:59:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:59:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2026-02-20T21:59:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:59:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:59:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:59:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:59:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:59:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:59:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:59:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:14.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:59:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:59:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2026-02-20T21:59:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:59:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:59:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:59:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:14.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2026-02-20T21:59:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:59:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_19 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:59:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:59:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:59:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_19 2026-02-20T21:59:14.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:59:14.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:14.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_19 2026-02-20T21:59:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:59:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:14.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:59:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:59:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:59:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:59:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:14.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_19 2026-02-20T21:59:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:59:14.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:59:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:59:14.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:59:14.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:59:14.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_19 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:59:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:59:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_19 2026-02-20T21:59:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_19 2026-02-20T21:59:15.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 122 2026-02-20T21:59:15.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_19 2026-02-20T21:59:15.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:15.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:59:15.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_19.status 2026-02-20T21:59:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:15.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_19.status 2026-02-20T21:59:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_19.status 2026-02-20T21:59:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_19.status 2026-02-20T21:59:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:15.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.status 2026-02-20T21:59:15.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_19 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_19.status 2026-02-20T21:59:15.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:15.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:15.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:15.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_19.status 2026-02-20T21:59:15.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_19 --format xml 2026-02-20T21:59:15.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:15.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:59:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=122 2026-02-20T21:59:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:15.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:59:15.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.status 2026-02-20T21:59:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.status 2026-02-20T21:59:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-20T21:59:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=53 2026-02-20T21:59:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 122 = 53 2026-02-20T21:59:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:59:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.status 2026-02-20T21:59:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.status 2026-02-20T21:59:16.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-20T21:59:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=53 2026-02-20T21:59:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 122 = 53 2026-02-20T21:59:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:59:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.status 2026-02-20T21:59:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.status 2026-02-20T21:59:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-20T21:59:17.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 122 = '' 2026-02-20T21:59:17.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:17.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_19 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.status 2026-02-20T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.status 2026-02-20T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_19 --format xml 2026-02-20T21:59:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:18.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:18.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=122 2026-02-20T21:59:18.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 122 = 122 2026-02-20T21:59:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2026-02-20T21:59:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:59:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:19.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2026-02-20T21:59:19.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_19': 2026-02-20T21:59:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:59:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:59:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:59:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0d0b72c08 2026-02-20T21:59:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0d0b72c08 2026-02-20T21:59:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:59:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:59:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:59:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:59:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:01 2026 2026-02-20T21:59:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:01 2026 2026-02-20T21:59:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:01 2026 2026-02-20T21:59:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:59:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:59:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:59:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 7494457f-5c66-44ae-b871-096697c31154 2026-02-20T21:59:19.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:59:19.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:19.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_19 snap 2026-02-20T21:59:19.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:19.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T21:59:19.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:19.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:59:19.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_19.export 2026-02-20T21:59:19.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.export 2026-02-20T21:59:19.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_19.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.export 2026-02-20T21:59:19.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_19.export 2026-02-20T21:59:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:20.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.export 2026-02-20T21:59:20.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_19.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.export 2026-02-20T21:59:20.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_19.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_19.export 2026-02-20T21:59:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:59:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_20 2026-02-20T21:59:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_20 snap 2026-02-20T21:59:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:20.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:20.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:21.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2026-02-20T21:59:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:59:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:59:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:59:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2026-02-20T21:59:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:59:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:59:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:59:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:59:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:59:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:59:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:59:22.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:59:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:59:22.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:22.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:22.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:22.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2026-02-20T21:59:22.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:59:22.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:22.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:59:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:59:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:59:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:59:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2026-02-20T21:59:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:59:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:22.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_20 2026-02-20T21:59:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:59:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:22.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:22.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:22.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:59:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:59:22.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:59:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_20 2026-02-20T21:59:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:59:22.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:22.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:22.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:22.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_20 2026-02-20T21:59:22.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:59:22.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:22.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:22.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:59:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:22.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:59:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:59:22.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:59:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_20 2026-02-20T21:59:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:59:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:59:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:59:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:59:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_20 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:59:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:59:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_20 2026-02-20T21:59:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:22.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_20 2026-02-20T21:59:23.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 124 2026-02-20T21:59:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_20 2026-02-20T21:59:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:23.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:23.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:23.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:23.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:59:23.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_20.status 2026-02-20T21:59:23.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_20.status 2026-02-20T21:59:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_20.status 2026-02-20T21:59:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_20.status 2026-02-20T21:59:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.status 2026-02-20T21:59:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_20 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_20.status 2026-02-20T21:59:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_20.status 2026-02-20T21:59:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_20 --format xml 2026-02-20T21:59:23.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:59:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=124 2026-02-20T21:59:23.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:23.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:23.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:59:23.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.status 2026-02-20T21:59:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.status 2026-02-20T21:59:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-20T21:59:23.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=54 2026-02-20T21:59:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 124 = 54 2026-02-20T21:59:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:59:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.status 2026-02-20T21:59:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.status 2026-02-20T21:59:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-20T21:59:24.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:24.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=54 2026-02-20T21:59:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 124 = 54 2026-02-20T21:59:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:59:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.status 2026-02-20T21:59:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.status 2026-02-20T21:59:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-20T21:59:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:25.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:25.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:59:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:59:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:59:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:59:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 124 = '' 2026-02-20T21:59:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:59:26.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_20 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.status 2026-02-20T21:59:26.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:26.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:26.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:26.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.status 2026-02-20T21:59:26.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_20 --format xml 2026-02-20T21:59:26.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:26.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=124 2026-02-20T21:59:26.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 124 = 124 2026-02-20T21:59:26.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:26.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2026-02-20T21:59:26.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:26.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:26.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:26.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:26.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:26.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:59:26.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:27.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2026-02-20T21:59:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_20': 2026-02-20T21:59:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:59:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:59:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:59:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b05d51a456 2026-02-20T21:59:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b05d51a456 2026-02-20T21:59:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:59:27.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:09 2026 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:09 2026 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:09 2026 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 919065ea-03af-418d-938d-c4f0d0f3196f 2026-02-20T21:59:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:59:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_20 snap 2026-02-20T21:59:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T21:59:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:59:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_20.export 2026-02-20T21:59:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.export 2026-02-20T21:59:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_20.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.export 2026-02-20T21:59:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_20.export 2026-02-20T21:59:28.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.export 2026-02-20T21:59:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:28.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_20.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.export 2026-02-20T21:59:28.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_20.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_20.export 2026-02-20T21:59:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:59:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_21 2026-02-20T21:59:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_21 snap 2026-02-20T21:59:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:59:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2026-02-20T21:59:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2026-02-20T21:59:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:59:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2026-02-20T21:59:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:59:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:59:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:59:30.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2026-02-20T21:59:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:59:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:59:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:59:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:59:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:59:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:30.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:59:30.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:59:30.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:59:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:59:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:59:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:59:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2026-02-20T21:59:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-20T21:59:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_21 2026-02-20T21:59:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:59:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:59:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:59:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:59:30.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:59:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_21 2026-02-20T21:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-20T21:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_21 2026-02-20T21:59:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:59:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:59:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:59:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:59:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:30.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:30.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_21 2026-02-20T21:59:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_21'] 2026-02-20T21:59:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_21 2026-02-20T21:59:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:59:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:59:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:59:30.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:59:30.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:59:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_21 2026-02-20T21:59:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:59:30.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:59:30.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:59:30.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:59:30.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:59:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:59:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:59:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_21 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_21 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_21 2026-02-20T21:59:31.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 126 2026-02-20T21:59:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_21 2026-02-20T21:59:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:59:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_21.status 2026-02-20T21:59:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_21.status 2026-02-20T21:59:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_21.status 2026-02-20T21:59:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_21.status 2026-02-20T21:59:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.status 2026-02-20T21:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_21 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_21.status 2026-02-20T21:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_21.status 2026-02-20T21:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_21 --format xml 2026-02-20T21:59:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:31.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:59:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=126 2026-02-20T21:59:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:31.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:59:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.status 2026-02-20T21:59:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.status 2026-02-20T21:59:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-20T21:59:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:31.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=55 2026-02-20T21:59:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 126 = 55 2026-02-20T21:59:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:59:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.status 2026-02-20T21:59:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.status 2026-02-20T21:59:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-20T21:59:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:32.384 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=55 2026-02-20T21:59:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 126 = 55 2026-02-20T21:59:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:59:33.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.status 2026-02-20T21:59:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.status 2026-02-20T21:59:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-20T21:59:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:59:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:59:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:59:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:59:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 126 = '' 2026-02-20T21:59:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:59:34.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_21 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.status 2026-02-20T21:59:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.status 2026-02-20T21:59:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_21 --format xml 2026-02-20T21:59:34.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=126 2026-02-20T21:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 126 = 126 2026-02-20T21:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2026-02-20T21:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:59:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:36.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2026-02-20T21:59:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_21': 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0716f7992 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0716f7992 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:22 2026 2026-02-20T21:59:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:22 2026 2026-02-20T21:59:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:22 2026 2026-02-20T21:59:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:59:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:59:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:59:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: ea30739c-7e9c-4234-88fe-ea59f7828681 2026-02-20T21:59:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:59:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_21 snap 2026-02-20T21:59:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T21:59:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:59:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_21.export 2026-02-20T21:59:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.export 2026-02-20T21:59:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_21.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.export 2026-02-20T21:59:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_21.export 2026-02-20T21:59:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.export 2026-02-20T21:59:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:36.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_21.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.export 2026-02-20T21:59:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_21.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_21.export 2026-02-20T21:59:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:59:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_22 2026-02-20T21:59:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_22 snap 2026-02-20T21:59:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:59:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2026-02-20T21:59:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:37.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2026-02-20T21:59:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:59:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:37.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:38.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2026-02-20T21:59:38.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:38.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:38.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:59:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:59:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:59:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:59:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:59:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:38.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:38.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:38.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:38.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:38.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:59:38.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:59:38.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:59:38.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2026-02-20T21:59:38.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:59:38.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:38.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_22 2026-02-20T21:59:38.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:38.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:38.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:59:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:59:38.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:59:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_22 2026-02-20T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_22 2026-02-20T21:59:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:59:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:59:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:59:38.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:59:38.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:59:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:59:38.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:38.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:38.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:38.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_22 2026-02-20T21:59:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:59:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:38.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_22 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:59:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:59:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:59:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:59:38.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:59:38.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:38.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:38.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:38.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_22 2026-02-20T21:59:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:59:38.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:59:38.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:59:38.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:59:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:59:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:59:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:59:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_22 2026-02-20T21:59:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:59:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:59:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_22 2026-02-20T21:59:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_22 2026-02-20T21:59:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 128 2026-02-20T21:59:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_22 2026-02-20T21:59:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:59:39.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_22.status 2026-02-20T21:59:39.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:39.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_22.status 2026-02-20T21:59:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_22.status 2026-02-20T21:59:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_22.status 2026-02-20T21:59:39.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.status 2026-02-20T21:59:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_22 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_22.status 2026-02-20T21:59:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_22.status 2026-02-20T21:59:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_22 --format xml 2026-02-20T21:59:39.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:39.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:59:39.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=128 2026-02-20T21:59:39.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:39.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:39.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:59:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.status 2026-02-20T21:59:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.status 2026-02-20T21:59:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-20T21:59:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=56 2026-02-20T21:59:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 128 = 56 2026-02-20T21:59:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:59:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.status 2026-02-20T21:59:40.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:40.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:40.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:40.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.status 2026-02-20T21:59:40.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-20T21:59:40.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:40.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=56 2026-02-20T21:59:40.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 128 = 56 2026-02-20T21:59:40.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:40.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:59:41.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.status 2026-02-20T21:59:41.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.status 2026-02-20T21:59:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-20T21:59:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:41.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:59:41.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:59:41.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:59:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:59:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 128 = '' 2026-02-20T21:59:41.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:41.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:59:42.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_22 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.status 2026-02-20T21:59:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:42.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.status 2026-02-20T21:59:42.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_22 --format xml 2026-02-20T21:59:42.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=128 2026-02-20T21:59:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 128 = 128 2026-02-20T21:59:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2026-02-20T21:59:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:59:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2026-02-20T21:59:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_22': 2026-02-20T21:59:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:59:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:59:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:59:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0b6a0bbbd 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0b6a0bbbd 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:32 2026 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:32 2026 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:32 2026 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:59:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 34a71a4e-36fa-455c-a493-c55de2c7e753 2026-02-20T21:59:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:59:44.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:44.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_22 snap 2026-02-20T21:59:44.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:44.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T21:59:44.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:44.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:59:44.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_22.export 2026-02-20T21:59:44.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.export 2026-02-20T21:59:44.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_22.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.export 2026-02-20T21:59:44.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_22.export 2026-02-20T21:59:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.export 2026-02-20T21:59:44.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_22.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.export 2026-02-20T21:59:44.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_22.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_22.export 2026-02-20T21:59:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:59:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_23 2026-02-20T21:59:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_23 snap 2026-02-20T21:59:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:45.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2026-02-20T21:59:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:59:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:59:46.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:59:46.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:59:46.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:46.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:59:46.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:59:46.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:59:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-20T21:59:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:59:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:46.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:46.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:46.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:46.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:46.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:59:46.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:59:46.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:59:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-02-20T21:59:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:59:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:46.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:46.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:46.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-02-20T21:59:46.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:46.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:59:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:59:46.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:59:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:46.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:46.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-02-20T21:59:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_23 2026-02-20T21:59:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:59:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:59:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:59:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:59:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_23 2026-02-20T21:59:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:59:47.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:59:47.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:59:47.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:59:47.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:59:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:59:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:59:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_23 2026-02-20T21:59:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_23 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_23 2026-02-20T21:59:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 130 2026-02-20T21:59:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_23 2026-02-20T21:59:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:59:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_23.status 2026-02-20T21:59:47.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:47.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_23.status 2026-02-20T21:59:47.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_23.status 2026-02-20T21:59:47.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_23.status 2026-02-20T21:59:47.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.status 2026-02-20T21:59:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_23 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_23.status 2026-02-20T21:59:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_23.status 2026-02-20T21:59:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_23 --format xml 2026-02-20T21:59:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:47.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:59:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=130 2026-02-20T21:59:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:47.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:59:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.status 2026-02-20T21:59:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.status 2026-02-20T21:59:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-20T21:59:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=57 2026-02-20T21:59:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 130 = 57 2026-02-20T21:59:47.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:47.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:59:48.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.status 2026-02-20T21:59:48.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:48.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:48.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.status 2026-02-20T21:59:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-20T21:59:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:48.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:48.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=57 2026-02-20T21:59:48.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 130 = 57 2026-02-20T21:59:48.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:48.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:59:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.status 2026-02-20T21:59:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.status 2026-02-20T21:59:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-20T21:59:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:49.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:49.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:49.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:59:49.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:59:49.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:59:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:59:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 130 = '' 2026-02-20T21:59:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:59:50.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_23 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.status 2026-02-20T21:59:50.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:50.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.status 2026-02-20T21:59:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_23 --format xml 2026-02-20T21:59:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:50.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=130 2026-02-20T21:59:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 130 = 130 2026-02-20T21:59:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2026-02-20T21:59:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:59:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2026-02-20T21:59:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_23': 2026-02-20T21:59:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:59:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:59:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0c9f2b570 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0c9f2b570 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:44 2026 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:44 2026 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:44 2026 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:59:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:59:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:59:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 19082fd1-184d-4597-821e-115e896cdf0f 2026-02-20T21:59:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:59:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_23 snap 2026-02-20T21:59:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T21:59:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:59:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_23.export 2026-02-20T21:59:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.export 2026-02-20T21:59:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_23.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.export 2026-02-20T21:59:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_23.export 2026-02-20T21:59:52.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.export 2026-02-20T21:59:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:59:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_23.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.export 2026-02-20T21:59:52.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_23.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_23.export 2026-02-20T21:59:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:59:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_24 2026-02-20T21:59:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_24 snap 2026-02-20T21:59:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:59:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2026-02-20T21:59:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2026-02-20T21:59:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T21:59:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:53.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2026-02-20T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:54.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T21:59:54.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T21:59:54.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T21:59:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2026-02-20T21:59:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T21:59:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T21:59:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T21:59:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T21:59:54.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T21:59:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:59:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T21:59:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:59:54.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T21:59:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:54.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:54.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:54.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T21:59:54.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:59:54.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:54.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:54.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T21:59:54.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T21:59:54.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:59:54.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:54.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:54.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:59:54.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2026-02-20T21:59:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-20T21:59:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:54.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_24 2026-02-20T21:59:54.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:54.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:54.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T21:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T21:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T21:59:54.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:59:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:59:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_24 2026-02-20T21:59:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-20T21:59:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_24 2026-02-20T21:59:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T21:59:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T21:59:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T21:59:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T21:59:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:59:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:59:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_24 2026-02-20T21:59:54.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-20T21:59:54.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:59:54.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T21:59:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_24 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T21:59:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T21:59:54.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T21:59:54.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T21:59:54.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:59:54.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:59:54.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T21:59:54.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T21:59:54.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T21:59:54.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:59:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:59:54.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_24 2026-02-20T21:59:55.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T21:59:55.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:59:55.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:59:55.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T21:59:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T21:59:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_24 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T21:59:55.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_24 2026-02-20T21:59:55.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:55.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:55.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:55.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_24 2026-02-20T21:59:55.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 132 2026-02-20T21:59:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_24 2026-02-20T21:59:55.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T21:59:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T21:59:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T21:59:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_24.status 2026-02-20T21:59:55.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_24.status 2026-02-20T21:59:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:55.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_24.status 2026-02-20T21:59:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_24.status 2026-02-20T21:59:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_24.status 2026-02-20T21:59:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T21:59:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.status 2026-02-20T21:59:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_24 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_24.status 2026-02-20T21:59:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T21:59:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_24.status 2026-02-20T21:59:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_24 --format xml 2026-02-20T21:59:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:55.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T21:59:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=132 2026-02-20T21:59:55.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T21:59:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.status 2026-02-20T21:59:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:55.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.status 2026-02-20T21:59:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-20T21:59:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:56.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=58 2026-02-20T21:59:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 132 = 58 2026-02-20T21:59:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T21:59:56.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.status 2026-02-20T21:59:56.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:56.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:56.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:56.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.status 2026-02-20T21:59:56.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-20T21:59:56.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:56.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=58 2026-02-20T21:59:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 132 = 58 2026-02-20T21:59:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:56.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T21:59:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.status 2026-02-20T21:59:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.status 2026-02-20T21:59:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-20T21:59:57.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:57.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T21:59:57.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T21:59:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T21:59:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T21:59:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T21:59:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 132 = '' 2026-02-20T21:59:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:59:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T21:59:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_24 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.status 2026-02-20T21:59:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.status 2026-02-20T21:59:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_24 --format xml 2026-02-20T21:59:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T21:59:58.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=132 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 132 = 132 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:59:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T21:59:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2026-02-20T21:59:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_24': 2026-02-20T21:59:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T21:59:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T21:59:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T21:59:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b019884b1e 2026-02-20T21:59:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b019884b1e 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:51:51 2026 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:51:51 2026 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:51:51 2026 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T21:59:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T21:59:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: af49b5c9-f8fe-41c3-a35c-c2ad69c2bd7f 2026-02-20T21:59:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T21:59:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T21:59:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_24 snap 2026-02-20T21:59:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T21:59:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T21:59:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T21:59:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T21:59:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_24.export 2026-02-20T21:59:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.export 2026-02-20T21:59:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_24.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.export 2026-02-20T21:59:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_24.export 2026-02-20T22:00:00.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.export 2026-02-20T22:00:00.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:00.274 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_24.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.export 2026-02-20T22:00:00.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_24.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_24.export 2026-02-20T22:00:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:00:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_25 2026-02-20T22:00:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_25 snap 2026-02-20T22:00:00.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:00:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2026-02-20T22:00:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:01.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2026-02-20T22:00:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:00:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2026-02-20T22:00:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:00:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:00:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:00:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2026-02-20T22:00:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:00:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:00:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:00:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:00:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:00:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:00:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:00:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:00:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2026-02-20T22:00:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:00:02.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:00:02.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:02.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:00:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:00:02.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:00:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:02.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2026-02-20T22:00:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-20T22:00:02.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:02.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_25 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:00:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:00:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:00:02.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:00:02.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:02.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:02.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:02.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_25 2026-02-20T22:00:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-20T22:00:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_25 2026-02-20T22:00:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:00:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:00:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:00:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:00:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:00:02.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:00:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:02.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_25 2026-02-20T22:00:02.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_25'] 2026-02-20T22:00:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:02.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:02.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_25 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:02.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:00:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:00:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:00:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_25 2026-02-20T22:00:03.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:00:03.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:00:03.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:00:03.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:00:03.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:00:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:00:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:00:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_25 2026-02-20T22:00:03.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_25 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:03.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_25 2026-02-20T22:00:03.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 134 2026-02-20T22:00:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_25 2026-02-20T22:00:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_25.status 2026-02-20T22:00:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_25.status 2026-02-20T22:00:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_25.status 2026-02-20T22:00:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_25.status 2026-02-20T22:00:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_25.status 2026-02-20T22:00:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.status 2026-02-20T22:00:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_25 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_25.status 2026-02-20T22:00:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_25.status 2026-02-20T22:00:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_25 --format xml 2026-02-20T22:00:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:03.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:00:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=134 2026-02-20T22:00:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:03.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:00:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.status 2026-02-20T22:00:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:04.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.status 2026-02-20T22:00:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-20T22:00:04.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=59 2026-02-20T22:00:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 134 = 59 2026-02-20T22:00:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:00:04.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.status 2026-02-20T22:00:04.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:04.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:04.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:04.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.status 2026-02-20T22:00:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-20T22:00:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 134 = '' 2026-02-20T22:00:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:00:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.status 2026-02-20T22:00:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.status 2026-02-20T22:00:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-20T22:00:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:05.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:05.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:05.500 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:05.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:05.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 134 = '' 2026-02-20T22:00:05.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:05.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:00:07.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_25 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.status 2026-02-20T22:00:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:07.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.status 2026-02-20T22:00:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_25 --format xml 2026-02-20T22:00:07.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=134 2026-02-20T22:00:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 134 = 134 2026-02-20T22:00:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2026-02-20T22:00:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:00:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:08.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2026-02-20T22:00:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_25': 2026-02-20T22:00:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:00:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:00:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:00:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0630dc047 2026-02-20T22:00:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0630dc047 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:01 2026 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:01 2026 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:01 2026 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:00:08.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:00:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 013e07d3-729c-41fe-bfbb-6be668748dac 2026-02-20T22:00:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:00:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_25 snap 2026-02-20T22:00:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:00:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:00:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_25.export 2026-02-20T22:00:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.export 2026-02-20T22:00:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_25.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.export 2026-02-20T22:00:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_25.export 2026-02-20T22:00:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.export 2026-02-20T22:00:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_25.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.export 2026-02-20T22:00:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_25.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_25.export 2026-02-20T22:00:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:00:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_26 2026-02-20T22:00:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_26 snap 2026-02-20T22:00:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:00:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2026-02-20T22:00:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2026-02-20T22:00:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:09.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:00:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2026-02-20T22:00:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:00:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:00:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:00:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2026-02-20T22:00:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:00:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:00:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:00:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:00:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:00:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:00:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:00:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:00:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2026-02-20T22:00:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:00:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:00:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:00:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:10.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2026-02-20T22:00:10.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T22:00:10.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:10.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_26 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:00:10.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:00:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:00:10.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:00:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_26 2026-02-20T22:00:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T22:00:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:10.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_26 2026-02-20T22:00:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:00:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:10.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:00:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:10.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:10.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:10.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:10.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:00:10.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:00:10.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:00:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_26 2026-02-20T22:00:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T22:00:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:10.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:10.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_26 2026-02-20T22:00:10.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:10.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:00:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:00:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:00:10.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:10.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_26 2026-02-20T22:00:11.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:00:11.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:00:11.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:00:11.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:00:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:00:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:00:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:00:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_26 2026-02-20T22:00:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:00:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:00:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_26 2026-02-20T22:00:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_26 2026-02-20T22:00:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 136 2026-02-20T22:00:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_26 2026-02-20T22:00:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_26.status 2026-02-20T22:00:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_26.status 2026-02-20T22:00:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_26.status 2026-02-20T22:00:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_26.status 2026-02-20T22:00:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_26.status 2026-02-20T22:00:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.status 2026-02-20T22:00:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_26 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_26.status 2026-02-20T22:00:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_26.status 2026-02-20T22:00:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_26 --format xml 2026-02-20T22:00:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:11.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:00:11.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=136 2026-02-20T22:00:11.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:00:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.status 2026-02-20T22:00:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.status 2026-02-20T22:00:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-20T22:00:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=60 2026-02-20T22:00:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 136 = 60 2026-02-20T22:00:12.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:12.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:00:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.status 2026-02-20T22:00:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.status 2026-02-20T22:00:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-20T22:00:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:12.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:12.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=60 2026-02-20T22:00:12.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 136 = 60 2026-02-20T22:00:12.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:12.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:00:13.143 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-20T22:00:13.143 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-20T22:00:13.143 INFO:tasks.rbd_mirror_thrash:waiting for 1.3855271592392138 secs before reviving daemons 2026-02-20T22:00:13.144 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:00:13.137+0000 7fb2d6f96640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 32083) UID: 1000 2026-02-20T22:00:13.274 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.status 2026-02-20T22:00:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.status 2026-02-20T22:00:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-20T22:00:13.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:13.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:13.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 136 = '' 2026-02-20T22:00:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:13.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:00:13.651 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T22:00:13.652 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T22:00:13.652 INFO:tasks.rbd_mirror_thrash:waiting for 1.4292608924111805 secs before reviving daemons 2026-02-20T22:00:13.653 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:00:13.645+0000 7f045b6b3640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 31396) UID: 1000 2026-02-20T22:00:14.529 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-20T22:00:14.530 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:00:14.530 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-20T22:00:14.530 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T22:00:14.531 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T22:00:14.533 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T22:00:14.533 INFO:tasks.rbd_mirror_thrash:waiting for 82.29334737291389 secs before thrashing 2026-02-20T22:00:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_26 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.status 2026-02-20T22:00:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.status 2026-02-20T22:00:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_26 --format xml 2026-02-20T22:00:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:14.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=136 2026-02-20T22:00:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 136 = 136 2026-02-20T22:00:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2026-02-20T22:00:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:14.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:14.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:14.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:14.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:14.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:00:14.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:15.082 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T22:00:15.082 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:00:15.082 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T22:00:15.082 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T22:00:15.082 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T22:00:15.083 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T22:00:15.083 INFO:tasks.rbd_mirror_thrash:waiting for 74.3814080772402 secs before thrashing 2026-02-20T22:00:15.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2026-02-20T22:00:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_26': 2026-02-20T22:00:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:00:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:00:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:00:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0ea306efe 2026-02-20T22:00:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0ea306efe 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:16 2026 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:16 2026 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:16 2026 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: eebea35a-dfd0-4c97-aade-b52b3cada4e8 2026-02-20T22:00:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:00:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_26 snap 2026-02-20T22:00:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:00:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:00:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_26.export 2026-02-20T22:00:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.export 2026-02-20T22:00:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_26.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.export 2026-02-20T22:00:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_26.export 2026-02-20T22:00:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.export 2026-02-20T22:00:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_26.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.export 2026-02-20T22:00:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_26.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_26.export 2026-02-20T22:00:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:00:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_27 2026-02-20T22:00:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_27 snap 2026-02-20T22:00:16.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:00:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2026-02-20T22:00:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2026-02-20T22:00:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:00:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2026-02-20T22:00:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:00:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:00:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:00:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2026-02-20T22:00:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:00:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:00:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:00:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:00:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:00:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:00:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:00:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:00:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2026-02-20T22:00:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:00:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:00:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:00:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:00:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:00:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:00:18.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:18.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:18.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:18.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2026-02-20T22:00:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T22:00:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:18.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:18.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:18.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:00:18.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:00:18.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:00:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27 2026-02-20T22:00:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T22:00:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:18.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_27 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:18.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:00:18.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:00:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:00:18.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:00:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_27 2026-02-20T22:00:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T22:00:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_27 2026-02-20T22:00:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:00:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:00:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:00:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:00:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:00:18.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:18.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:18.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:18.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_27 2026-02-20T22:00:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:00:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:00:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:00:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:00:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:00:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:00:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:00:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_27 2026-02-20T22:00:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_27 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_27 2026-02-20T22:00:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 138 2026-02-20T22:00:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_27 2026-02-20T22:00:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_27.status 2026-02-20T22:00:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_27.status 2026-02-20T22:00:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_27.status 2026-02-20T22:00:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_27.status 2026-02-20T22:00:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_27.status 2026-02-20T22:00:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:19.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.status 2026-02-20T22:00:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_27 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_27.status 2026-02-20T22:00:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_27.status 2026-02-20T22:00:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_27 --format xml 2026-02-20T22:00:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:19.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:00:19.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=138 2026-02-20T22:00:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:19.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:00:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.status 2026-02-20T22:00:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.status 2026-02-20T22:00:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-20T22:00:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:20.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=61 2026-02-20T22:00:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 138 = 61 2026-02-20T22:00:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:00:20.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.status 2026-02-20T22:00:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.status 2026-02-20T22:00:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-20T22:00:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=61 2026-02-20T22:00:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 138 = 61 2026-02-20T22:00:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:00:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.status 2026-02-20T22:00:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.status 2026-02-20T22:00:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-20T22:00:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 138 = '' 2026-02-20T22:00:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:00:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_27 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.status 2026-02-20T22:00:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.status 2026-02-20T22:00:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_27 --format xml 2026-02-20T22:00:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:22.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=138 2026-02-20T22:00:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 138 = 138 2026-02-20T22:00:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2026-02-20T22:00:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:00:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:23.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2026-02-20T22:00:24.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_27': 2026-02-20T22:00:24.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b02a90f95b 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b02a90f95b 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:22 2026 2026-02-20T22:00:24.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:22 2026 2026-02-20T22:00:24.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:22 2026 2026-02-20T22:00:24.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:00:24.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:00:24.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:00:24.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 1b8e35e9-ec45-44f4-a0d7-dbb42eabaaa7 2026-02-20T22:00:24.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:00:24.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:24.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_27 snap 2026-02-20T22:00:24.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:00:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:00:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_27.export 2026-02-20T22:00:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.export 2026-02-20T22:00:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_27.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.export 2026-02-20T22:00:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_27.export 2026-02-20T22:00:24.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:24.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.export 2026-02-20T22:00:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:24.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_27.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.export 2026-02-20T22:00:24.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_27.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_27.export 2026-02-20T22:00:24.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:24.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:00:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_28 2026-02-20T22:00:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_28 snap 2026-02-20T22:00:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:24.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:00:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2026-02-20T22:00:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:00:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:00:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:00:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2026-02-20T22:00:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:00:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:00:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:00:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:00:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:00:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:00:27.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:00:27.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:00:27.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:27.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-20T22:00:27.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:00:27.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:27.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:00:27.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:27.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:00:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:00:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:00:27.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:27.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:27.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:27.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-20T22:00:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T22:00:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-20T22:00:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:00:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:00:27.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:00:27.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:00:27.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:00:27.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:00:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-20T22:00:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T22:00:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:27.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:27.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:27.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:00:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:00:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:00:27.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-20T22:00:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T22:00:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_28 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:00:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:00:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:28.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_28 2026-02-20T22:00:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:00:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:00:28.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:00:28.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:00:28.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_28 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_28 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_28 2026-02-20T22:00:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 140 2026-02-20T22:00:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_28 2026-02-20T22:00:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:28.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:28.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:28.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:28.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:28.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_28.status 2026-02-20T22:00:28.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_28.status 2026-02-20T22:00:28.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:28.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_28.status 2026-02-20T22:00:28.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_28.status 2026-02-20T22:00:28.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_28.status 2026-02-20T22:00:28.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.status 2026-02-20T22:00:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_28 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_28.status 2026-02-20T22:00:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_28.status 2026-02-20T22:00:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_28 --format xml 2026-02-20T22:00:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:28.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:00:28.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=140 2026-02-20T22:00:28.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:28.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:28.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:00:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.status 2026-02-20T22:00:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.status 2026-02-20T22:00:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-20T22:00:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=62 2026-02-20T22:00:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 140 = 62 2026-02-20T22:00:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:00:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.status 2026-02-20T22:00:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.status 2026-02-20T22:00:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-20T22:00:29.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:29.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=62 2026-02-20T22:00:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 140 = 62 2026-02-20T22:00:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:00:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.status 2026-02-20T22:00:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.status 2026-02-20T22:00:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-20T22:00:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 140 = '' 2026-02-20T22:00:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:30.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:00:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_28 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.status 2026-02-20T22:00:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.status 2026-02-20T22:00:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_28 --format xml 2026-02-20T22:00:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=140 2026-02-20T22:00:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 140 = 140 2026-02-20T22:00:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2026-02-20T22:00:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:00:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:32.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2026-02-20T22:00:33.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_28': 2026-02-20T22:00:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:00:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:00:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:00:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b02b1e9655 2026-02-20T22:00:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b02b1e9655 2026-02-20T22:00:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:00:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:00:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:00:33.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:00:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:35 2026 2026-02-20T22:00:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:35 2026 2026-02-20T22:00:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:35 2026 2026-02-20T22:00:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:00:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:00:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:00:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: ab5b8a6b-3393-4121-92ba-22af9b8d3ce4 2026-02-20T22:00:33.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:00:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_28 snap 2026-02-20T22:00:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:00:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:00:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_28.export 2026-02-20T22:00:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.export 2026-02-20T22:00:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_28.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.export 2026-02-20T22:00:33.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_28.export 2026-02-20T22:00:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:33.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.export 2026-02-20T22:00:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_28.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.export 2026-02-20T22:00:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_28.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_28.export 2026-02-20T22:00:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:00:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_29 2026-02-20T22:00:33.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_29 snap 2026-02-20T22:00:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:33.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:00:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2026-02-20T22:00:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2026-02-20T22:00:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:00:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:35.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2026-02-20T22:00:35.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:35.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:35.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:35.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:00:35.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:00:35.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:00:35.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2026-02-20T22:00:35.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:00:35.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:00:35.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:00:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:00:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:00:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:00:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:00:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:00:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:00:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:00:35.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:35.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:35.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:35.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2026-02-20T22:00:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T22:00:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_29 2026-02-20T22:00:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:00:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:00:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:00:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:35.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:00:35.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:00:35.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:35.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:35.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:35.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_29 2026-02-20T22:00:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T22:00:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_29 2026-02-20T22:00:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:00:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:00:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:00:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:00:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:00:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_29 2026-02-20T22:00:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T22:00:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_29 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:00:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:00:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:00:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:00:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_29 2026-02-20T22:00:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:00:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:00:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:00:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:00:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_29 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:00:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_29 2026-02-20T22:00:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_29 2026-02-20T22:00:36.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 142 2026-02-20T22:00:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_29 2026-02-20T22:00:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:36.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_29.status 2026-02-20T22:00:36.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_29.status 2026-02-20T22:00:36.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:36.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_29.status 2026-02-20T22:00:36.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_29.status 2026-02-20T22:00:36.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_29.status 2026-02-20T22:00:36.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.status 2026-02-20T22:00:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_29 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_29.status 2026-02-20T22:00:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_29.status 2026-02-20T22:00:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_29 --format xml 2026-02-20T22:00:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:00:36.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=142 2026-02-20T22:00:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:00:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.status 2026-02-20T22:00:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.status 2026-02-20T22:00:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-20T22:00:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:37.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=63 2026-02-20T22:00:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 142 = 63 2026-02-20T22:00:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:00:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.status 2026-02-20T22:00:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.status 2026-02-20T22:00:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-20T22:00:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=63 2026-02-20T22:00:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 142 = 63 2026-02-20T22:00:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:00:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.status 2026-02-20T22:00:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.status 2026-02-20T22:00:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-20T22:00:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:38.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 142 = '' 2026-02-20T22:00:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:00:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_29 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.status 2026-02-20T22:00:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.status 2026-02-20T22:00:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_29 --format xml 2026-02-20T22:00:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:40.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=142 2026-02-20T22:00:40.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 142 = 142 2026-02-20T22:00:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2026-02-20T22:00:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:00:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2026-02-20T22:00:41.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_29': 2026-02-20T22:00:41.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:00:41.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:00:41.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:00:41.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0908e4dda 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0908e4dda 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:41 2026 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:41 2026 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:41 2026 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:00:41.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:00:41.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 2d03f9ea-eefd-44d9-b8bd-e230886e2bc8 2026-02-20T22:00:41.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:00:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_29 snap 2026-02-20T22:00:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:00:41.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:41.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:00:41.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_29.export 2026-02-20T22:00:41.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.export 2026-02-20T22:00:41.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_29.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.export 2026-02-20T22:00:41.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_29.export 2026-02-20T22:00:41.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:41.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.export 2026-02-20T22:00:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_29.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.export 2026-02-20T22:00:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_29.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_29.export 2026-02-20T22:00:41.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:00:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_30 2026-02-20T22:00:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_30 snap 2026-02-20T22:00:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:00:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2026-02-20T22:00:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2026-02-20T22:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:44.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2026-02-20T22:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:00:44.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:44.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:00:44.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:00:44.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:00:44.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:00:44.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:00:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:00:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:00:44.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:44.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-20T22:00:44.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T22:00:44.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:44.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:00:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:00:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:00:44.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-20T22:00:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T22:00:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-20T22:00:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:00:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:44.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:44.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:00:44.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:00:44.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:00:44.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:44.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:44.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:44.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-20T22:00:44.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T22:00:44.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:44.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-20T22:00:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:00:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:00:44.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:00:44.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:00:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:00:44.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:00:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:44.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-20T22:00:44.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:00:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:00:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:00:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:00:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:00:44.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_30 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:00:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_30 2026-02-20T22:00:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_30 2026-02-20T22:00:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 144 2026-02-20T22:00:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_30 2026-02-20T22:00:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:45.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_30.status 2026-02-20T22:00:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_30.status 2026-02-20T22:00:45.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:45.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_30.status 2026-02-20T22:00:45.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_30.status 2026-02-20T22:00:45.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_30.status 2026-02-20T22:00:45.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.status 2026-02-20T22:00:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_30 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_30.status 2026-02-20T22:00:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_30.status 2026-02-20T22:00:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_30 --format xml 2026-02-20T22:00:45.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:00:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=144 2026-02-20T22:00:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:00:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.status 2026-02-20T22:00:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.status 2026-02-20T22:00:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-20T22:00:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 144 = '' 2026-02-20T22:00:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:00:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.status 2026-02-20T22:00:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.status 2026-02-20T22:00:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-20T22:00:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:46.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:46.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 144 = '' 2026-02-20T22:00:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:00:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_30 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.status 2026-02-20T22:00:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:47.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.status 2026-02-20T22:00:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_30 --format xml 2026-02-20T22:00:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:47.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=144 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 144 = 144 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:00:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:48.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2026-02-20T22:00:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_30': 2026-02-20T22:00:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:00:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:00:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:00:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0da74c454 2026-02-20T22:00:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0da74c454 2026-02-20T22:00:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:00:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:47 2026 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:47 2026 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:47 2026 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: d649a580-cc10-448a-bdd7-2ee6987b9a69 2026-02-20T22:00:48.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:00:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_30 snap 2026-02-20T22:00:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:00:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:00:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_30.export 2026-02-20T22:00:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.export 2026-02-20T22:00:48.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_30.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.export 2026-02-20T22:00:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_30.export 2026-02-20T22:00:48.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:48.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.export 2026-02-20T22:00:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_30.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.export 2026-02-20T22:00:48.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_30.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_30.export 2026-02-20T22:00:48.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:48.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:00:48.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_31 2026-02-20T22:00:48.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_31 snap 2026-02-20T22:00:48.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:48.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:48.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:48.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:48.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:00:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2026-02-20T22:00:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2026-02-20T22:00:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:00:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2026-02-20T22:00:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:00:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:00:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:00:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:00:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2026-02-20T22:00:51.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:00:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:00:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:00:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:00:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:51.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:00:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:00:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:00:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2026-02-20T22:00:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:00:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:00:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:00:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:51.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:00:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:00:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:00:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:51.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:51.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:51.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2026-02-20T22:00:51.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T22:00:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:00:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:00:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:00:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:00:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:51.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:51.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:51.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2026-02-20T22:00:51.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T22:00:51.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:51.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_31 2026-02-20T22:00:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:00:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:00:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:00:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:51.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:51.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:51.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_31 2026-02-20T22:00:51.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T22:00:51.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:51.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_31 2026-02-20T22:00:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:00:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:00:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:00:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:00:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:00:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_31 2026-02-20T22:00:51.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:00:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:00:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:00:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:00:51.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_31 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:00:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:00:51.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_31 2026-02-20T22:00:51.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:51.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:51.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:51.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_31 2026-02-20T22:00:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 146 2026-02-20T22:00:52.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_31 2026-02-20T22:00:52.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:52.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:52.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:52.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:52.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:52.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_31.status 2026-02-20T22:00:52.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_31.status 2026-02-20T22:00:52.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:52.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_31.status 2026-02-20T22:00:52.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_31.status 2026-02-20T22:00:52.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_31.status 2026-02-20T22:00:52.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:52.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.status 2026-02-20T22:00:52.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_31 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_31.status 2026-02-20T22:00:52.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:52.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:52.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:52.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_31.status 2026-02-20T22:00:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_31 --format xml 2026-02-20T22:00:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:00:52.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=146 2026-02-20T22:00:52.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:00:52.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.status 2026-02-20T22:00:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.status 2026-02-20T22:00:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2026-02-20T22:00:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:52.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:52.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:52.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:52.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 146 = '' 2026-02-20T22:00:52.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:52.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:00:53.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.status 2026-02-20T22:00:53.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:53.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.status 2026-02-20T22:00:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2026-02-20T22:00:53.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:00:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:00:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:00:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:00:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 146 = '' 2026-02-20T22:00:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:00:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_31 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.status 2026-02-20T22:00:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.status 2026-02-20T22:00:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_31 --format xml 2026-02-20T22:00:54.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=146 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 146 = 146 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:00:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2026-02-20T22:00:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_31': 2026-02-20T22:00:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:00:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:00:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:00:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b06094b563 2026-02-20T22:00:55.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b06094b563 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:52:55 2026 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:52:55 2026 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:52:55 2026 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:00:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 28720fa9-2640-4d05-b3cb-cf921814d23d 2026-02-20T22:00:55.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:00:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_31 snap 2026-02-20T22:00:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:55.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:00:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:00:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_31.export 2026-02-20T22:00:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.export 2026-02-20T22:00:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_31.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.export 2026-02-20T22:00:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_31.export 2026-02-20T22:00:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:55.335 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.export 2026-02-20T22:00:55.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:00:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_31.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.export 2026-02-20T22:00:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_31.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_31.export 2026-02-20T22:00:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:00:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_32 2026-02-20T22:00:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_32 snap 2026-02-20T22:00:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:00:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:00:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:00:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2026-02-20T22:00:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:00:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2026-02-20T22:00:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:00:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:00:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:00:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:57.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:00:58.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2026-02-20T22:00:58.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:00:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:00:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:00:58.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:00:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2026-02-20T22:00:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:00:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:00:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:00:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:00:58.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:00:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:00:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:00:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:00:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:00:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2026-02-20T22:00:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:00:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:00:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:00:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:00:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:00:58.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:58.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:00:58.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2026-02-20T22:00:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T22:00:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2026-02-20T22:00:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:00:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:00:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:00:58.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:00:58.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:00:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:00:58.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2026-02-20T22:00:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T22:00:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:58.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_32 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:58.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:00:58.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:00:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:00:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:00:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:00:58.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:00:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:58.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:58.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:00:58.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_32 2026-02-20T22:00:58.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T22:00:58.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:00:58.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_32 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:00:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:00:58.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:00:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:00:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:00:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:00:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:00:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:00:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:00:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:00:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:00:58.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:00:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:00:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_32 2026-02-20T22:00:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:00:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:00:58.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:00:58.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:00:58.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:00:58.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:00:58.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_32 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_32 2026-02-20T22:00:58.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:00:58.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_32 2026-02-20T22:00:59.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 148 2026-02-20T22:00:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_32 2026-02-20T22:00:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:00:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:00:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:00:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:00:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_32.status 2026-02-20T22:00:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_32.status 2026-02-20T22:00:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:59.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_32.status 2026-02-20T22:00:59.600 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_32.status 2026-02-20T22:00:59.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_32.status 2026-02-20T22:00:59.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:00:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.status 2026-02-20T22:00:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_32 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_32.status 2026-02-20T22:00:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:00:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:00:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_32.status 2026-02-20T22:00:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_32 --format xml 2026-02-20T22:00:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:59.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:00:59.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=148 2026-02-20T22:00:59.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:00:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:00:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.status 2026-02-20T22:00:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:00:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:00:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:00:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.status 2026-02-20T22:00:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-20T22:00:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:00:59.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:00:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=66 2026-02-20T22:00:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 148 = 66 2026-02-20T22:00:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:00:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:01:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.status 2026-02-20T22:01:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:01:00.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.status 2026-02-20T22:01:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-20T22:01:00.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:00.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=66 2026-02-20T22:01:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 148 = 66 2026-02-20T22:01:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:00.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:01:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.status 2026-02-20T22:01:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:01:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.status 2026-02-20T22:01:01.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-20T22:01:01.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:01.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:01.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:01.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:01:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:01:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:01:01.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:01:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 148 = '' 2026-02-20T22:01:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:01.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:01:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_32 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.status 2026-02-20T22:01:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:01:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.status 2026-02-20T22:01:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_32 --format xml 2026-02-20T22:01:02.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=148 2026-02-20T22:01:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 148 = 148 2026-02-20T22:01:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2026-02-20T22:01:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:02.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:01:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:01:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2026-02-20T22:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_32': 2026-02-20T22:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b08e75836b 2026-02-20T22:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b08e75836b 2026-02-20T22:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:01:03.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:10 2026 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:10 2026 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:10 2026 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 6f7f0c8e-e585-45a6-922c-3d3abe3d38f8 2026-02-20T22:01:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_32 snap 2026-02-20T22:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_32.export 2026-02-20T22:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.export 2026-02-20T22:01:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_32.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.export 2026-02-20T22:01:03.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_32.export 2026-02-20T22:01:03.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:03.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.export 2026-02-20T22:01:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:04.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_32.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.export 2026-02-20T22:01:04.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_32.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_32.export 2026-02-20T22:01:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:01:04.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_33 2026-02-20T22:01:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_33 snap 2026-02-20T22:01:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:01:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2026-02-20T22:01:05.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2026-02-20T22:01:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:06.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:06.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:01:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:01:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:01:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2026-02-20T22:01:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T22:01:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2026-02-20T22:01:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:01:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:01:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:01:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:01:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:01:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:01:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:01:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2026-02-20T22:01:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T22:01:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:06.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2026-02-20T22:01:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:01:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:01:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:01:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:01:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2026-02-20T22:01:06.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T22:01:06.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:06.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_33 2026-02-20T22:01:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:01:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:01:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:01:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:01:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:01:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:01:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:01:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_33 2026-02-20T22:01:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:01:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:01:06.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:01:06.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:01:06.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_33 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_33 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:06.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_33 2026-02-20T22:01:07.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 150 2026-02-20T22:01:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_33 2026-02-20T22:01:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:07.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_33.status 2026-02-20T22:01:07.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_33.status 2026-02-20T22:01:07.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:07.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_33.status 2026-02-20T22:01:07.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_33.status 2026-02-20T22:01:07.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_33.status 2026-02-20T22:01:07.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.status 2026-02-20T22:01:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_33 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_33.status 2026-02-20T22:01:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_33.status 2026-02-20T22:01:07.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_33 --format xml 2026-02-20T22:01:07.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:01:07.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=150 2026-02-20T22:01:07.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:07.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:07.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:01:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.status 2026-02-20T22:01:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.status 2026-02-20T22:01:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-20T22:01:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=67 2026-02-20T22:01:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 150 = 67 2026-02-20T22:01:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:01:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.status 2026-02-20T22:01:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.status 2026-02-20T22:01:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-20T22:01:08.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:08.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=67 2026-02-20T22:01:08.531 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 150 = 67 2026-02-20T22:01:08.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:08.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:01:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.status 2026-02-20T22:01:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.status 2026-02-20T22:01:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-20T22:01:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:09.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:09.364 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:01:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:01:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 150 = '' 2026-02-20T22:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:09.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:01:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_33 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.status 2026-02-20T22:01:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.status 2026-02-20T22:01:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_33 --format xml 2026-02-20T22:01:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:11.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=150 2026-02-20T22:01:11.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 150 = 150 2026-02-20T22:01:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2026-02-20T22:01:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:01:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:12.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2026-02-20T22:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_33': 2026-02-20T22:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b072bbc2fb 2026-02-20T22:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b072bbc2fb 2026-02-20T22:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:01:12.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:18 2026 2026-02-20T22:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:18 2026 2026-02-20T22:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:18 2026 2026-02-20T22:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 9a59b858-296a-4d6b-9785-36f62034ac7a 2026-02-20T22:01:12.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:01:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_33 snap 2026-02-20T22:01:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:01:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:01:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_33.export 2026-02-20T22:01:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.export 2026-02-20T22:01:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_33.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.export 2026-02-20T22:01:12.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_33.export 2026-02-20T22:01:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:12.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.export 2026-02-20T22:01:12.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:12.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_33.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.export 2026-02-20T22:01:12.409 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_33.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_33.export 2026-02-20T22:01:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:12.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:01:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_34 2026-02-20T22:01:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_34 snap 2026-02-20T22:01:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:12.456 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2026-02-20T22:01:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:01:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:01:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:01:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2026-02-20T22:01:15.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:01:15.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:01:15.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:15.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:01:15.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:01:15.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:01:15.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:15.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:01:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:01:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:01:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2026-02-20T22:01:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:01:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:01:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:01:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:01:15.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:01:15.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:15.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:15.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:15.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2026-02-20T22:01:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-20T22:01:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_34 2026-02-20T22:01:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:01:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:01:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:01:15.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:15.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:01:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:01:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:01:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:01:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:01:15.703 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_34 2026-02-20T22:01:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-20T22:01:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:15.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_34 2026-02-20T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:15.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:15.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:01:15.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:01:15.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:01:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_34 2026-02-20T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_34'] 2026-02-20T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:15.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:15.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_34 2026-02-20T22:01:15.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:01:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:01:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:01:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:01:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:01:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:01:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:01:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:01:15.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:15.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_34 2026-02-20T22:01:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:01:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:01:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:01:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:01:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:01:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:01:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:01:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_34 2026-02-20T22:01:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_34 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_34 2026-02-20T22:01:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 152 2026-02-20T22:01:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_34 2026-02-20T22:01:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_34.status 2026-02-20T22:01:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_34.status 2026-02-20T22:01:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_34.status 2026-02-20T22:01:16.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_34.status 2026-02-20T22:01:16.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_34.status 2026-02-20T22:01:16.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:16.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.status 2026-02-20T22:01:16.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_34 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_34.status 2026-02-20T22:01:16.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:16.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:16.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:16.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_34.status 2026-02-20T22:01:16.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_34 --format xml 2026-02-20T22:01:16.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:01:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=152 2026-02-20T22:01:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:01:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.status 2026-02-20T22:01:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.status 2026-02-20T22:01:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-20T22:01:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:16.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:16.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=68 2026-02-20T22:01:16.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 152 = 68 2026-02-20T22:01:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:01:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.status 2026-02-20T22:01:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.status 2026-02-20T22:01:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-20T22:01:17.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:17.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:17.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=68 2026-02-20T22:01:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 152 = 68 2026-02-20T22:01:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:17.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:01:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.status 2026-02-20T22:01:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.status 2026-02-20T22:01:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-20T22:01:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:01:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:01:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:01:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:01:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 152 = '' 2026-02-20T22:01:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:01:19.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_34 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.status 2026-02-20T22:01:19.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:19.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.status 2026-02-20T22:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_34 --format xml 2026-02-20T22:01:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=152 2026-02-20T22:01:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 152 = 152 2026-02-20T22:01:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2026-02-20T22:01:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:01:19.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:20.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2026-02-20T22:01:20.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_34': 2026-02-20T22:01:20.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:01:20.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:01:20.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:01:20.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b082f0f4d8 2026-02-20T22:01:20.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b082f0f4d8 2026-02-20T22:01:20.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:25 2026 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:25 2026 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:25 2026 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 843abf52-674b-43d7-b959-4287ddcbd20a 2026-02-20T22:01:20.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:01:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_34 snap 2026-02-20T22:01:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:01:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:01:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_34.export 2026-02-20T22:01:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.export 2026-02-20T22:01:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_34.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.export 2026-02-20T22:01:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_34.export 2026-02-20T22:01:20.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.export 2026-02-20T22:01:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_34.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.export 2026-02-20T22:01:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_34.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_34.export 2026-02-20T22:01:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:01:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_35 2026-02-20T22:01:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_35 snap 2026-02-20T22:01:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:01:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:22.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:23.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2026-02-20T22:01:23.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:23.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:23.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:23.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:01:23.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:01:23.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:01:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2026-02-20T22:01:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:01:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:01:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:01:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:01:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:01:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:23.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:01:23.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:01:23.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:01:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:01:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:01:23.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:23.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:23.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:23.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2026-02-20T22:01:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-20T22:01:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:23.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_35 2026-02-20T22:01:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:01:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:01:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:23.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:01:23.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:01:23.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:01:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_35 2026-02-20T22:01:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-20T22:01:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:23.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:23.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:23.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_35 2026-02-20T22:01:23.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:01:23.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:23.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:01:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:01:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:01:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:01:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:23.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_35 2026-02-20T22:01:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_35'] 2026-02-20T22:01:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_35 2026-02-20T22:01:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:01:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:01:23.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:23.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:01:23.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:01:23.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:01:23.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:23.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:23.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:23.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_35 2026-02-20T22:01:23.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:01:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:01:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:01:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:01:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:01:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:01:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:01:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_35 2026-02-20T22:01:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_35 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:23.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_35 2026-02-20T22:01:24.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 154 2026-02-20T22:01:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_35 2026-02-20T22:01:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:24.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_35.status 2026-02-20T22:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_35.status 2026-02-20T22:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_35.status 2026-02-20T22:01:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_35.status 2026-02-20T22:01:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_35.status 2026-02-20T22:01:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.status 2026-02-20T22:01:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_35 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_35.status 2026-02-20T22:01:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_35.status 2026-02-20T22:01:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_35 --format xml 2026-02-20T22:01:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:24.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:01:24.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=154 2026-02-20T22:01:24.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:24.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:24.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:01:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.status 2026-02-20T22:01:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.status 2026-02-20T22:01:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-20T22:01:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:24.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=69 2026-02-20T22:01:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 154 = 69 2026-02-20T22:01:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:01:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.status 2026-02-20T22:01:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.status 2026-02-20T22:01:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-20T22:01:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:25.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=69 2026-02-20T22:01:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 154 = 69 2026-02-20T22:01:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:01:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.status 2026-02-20T22:01:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.status 2026-02-20T22:01:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-20T22:01:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:01:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:01:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:01:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:01:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 154 = '' 2026-02-20T22:01:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:26.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:01:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_35 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.status 2026-02-20T22:01:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.status 2026-02-20T22:01:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_35 --format xml 2026-02-20T22:01:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:27.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=154 2026-02-20T22:01:27.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 154 = 154 2026-02-20T22:01:27.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2026-02-20T22:01:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:01:27.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2026-02-20T22:01:28.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_35': 2026-02-20T22:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b016f5320f 2026-02-20T22:01:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b016f5320f 2026-02-20T22:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:35 2026 2026-02-20T22:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:35 2026 2026-02-20T22:01:28.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:35 2026 2026-02-20T22:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 15ffbe72-993b-43ba-9ff0-25518ecb6ae4 2026-02-20T22:01:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:01:28.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_35 snap 2026-02-20T22:01:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:01:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:01:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_35.export 2026-02-20T22:01:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.export 2026-02-20T22:01:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_35.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.export 2026-02-20T22:01:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_35.export 2026-02-20T22:01:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:29.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.export 2026-02-20T22:01:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:29.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_35.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.export 2026-02-20T22:01:29.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_35.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_35.export 2026-02-20T22:01:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:01:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_36 2026-02-20T22:01:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_36 snap 2026-02-20T22:01:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:29.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:29.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:01:29.465 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T22:01:29.466 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T22:01:29.466 INFO:tasks.rbd_mirror_thrash:waiting for 9.792814741191204 secs before reviving daemons 2026-02-20T22:01:29.466 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:01:29.463+0000 7fc1be4a7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 39552) UID: 1000 2026-02-20T22:01:30.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2026-02-20T22:01:30.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:30.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:30.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:30.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2026-02-20T22:01:30.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:01:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:31.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:01:31.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:31.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:01:31.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:01:31.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:01:31.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:31.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:01:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:01:31.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:31.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:01:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:01:31.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:31.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-20T22:01:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-20T22:01:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_36 2026-02-20T22:01:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:01:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:01:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:01:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:01:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:01:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:01:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:01:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:01:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:01:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:01:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:01:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:01:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:31.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:31.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:31.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_36 2026-02-20T22:01:31.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-20T22:01:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_36 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:01:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:01:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:01:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_36 2026-02-20T22:01:31.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_36'] 2026-02-20T22:01:31.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:31.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_36 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:31.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:01:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:01:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:01:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_36 2026-02-20T22:01:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:01:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:01:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:01:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:01:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:01:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_36 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:01:31.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_36 2026-02-20T22:01:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:31.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_36 2026-02-20T22:01:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 156 2026-02-20T22:01:32.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_36 2026-02-20T22:01:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:32.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_36.status 2026-02-20T22:01:32.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_36.status 2026-02-20T22:01:32.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:32.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_36.status 2026-02-20T22:01:32.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_36.status 2026-02-20T22:01:32.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_36.status 2026-02-20T22:01:32.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.status 2026-02-20T22:01:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_36 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_36.status 2026-02-20T22:01:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_36.status 2026-02-20T22:01:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_36 --format xml 2026-02-20T22:01:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:32.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:01:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=156 2026-02-20T22:01:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:01:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.status 2026-02-20T22:01:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.status 2026-02-20T22:01:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-20T22:01:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=70 2026-02-20T22:01:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 156 = 70 2026-02-20T22:01:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:01:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.status 2026-02-20T22:01:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.status 2026-02-20T22:01:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-20T22:01:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:33.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=70 2026-02-20T22:01:33.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 156 = 70 2026-02-20T22:01:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:01:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.status 2026-02-20T22:01:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.status 2026-02-20T22:01:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-20T22:01:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:34.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:01:34.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:01:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:01:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:01:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 156 = '' 2026-02-20T22:01:34.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:34.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:01:35.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_36 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.status 2026-02-20T22:01:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.status 2026-02-20T22:01:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_36 --format xml 2026-02-20T22:01:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:35.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:35.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=156 2026-02-20T22:01:35.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 156 = 156 2026-02-20T22:01:35.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2026-02-20T22:01:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:01:35.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:36.827 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-20T22:01:36.827 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-20T22:01:36.828 INFO:tasks.rbd_mirror_thrash:waiting for 8.329665114159415 secs before reviving daemons 2026-02-20T22:01:36.828 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:01:36.823+0000 7f9ae2aba640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 39437) UID: 1000 2026-02-20T22:01:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2026-02-20T22:01:36.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_36': 2026-02-20T22:01:36.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:01:36.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:01:36.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:01:36.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b01fdd4220 2026-02-20T22:01:36.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b01fdd4220 2026-02-20T22:01:36.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:48 2026 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:48 2026 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:48 2026 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: cb656abb-b533-4a24-8070-aceb29417b9d 2026-02-20T22:01:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:01:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_36 snap 2026-02-20T22:01:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:01:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:01:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_36.export 2026-02-20T22:01:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.export 2026-02-20T22:01:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_36.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.export 2026-02-20T22:01:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_36.export 2026-02-20T22:01:37.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.export 2026-02-20T22:01:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_36.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.export 2026-02-20T22:01:37.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_36.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_36.export 2026-02-20T22:01:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:01:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_37 2026-02-20T22:01:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_37 snap 2026-02-20T22:01:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:01:38.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2026-02-20T22:01:38.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:39.260 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T22:01:39.260 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:01:39.260 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T22:01:39.260 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T22:01:39.260 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T22:01:39.262 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T22:01:39.262 INFO:tasks.rbd_mirror_thrash:waiting for 87.32919090991491 secs before thrashing 2026-02-20T22:01:39.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2026-02-20T22:01:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:01:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:01:39.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:01:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2026-02-20T22:01:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:01:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:01:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:01:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:01:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:01:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:01:39.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:01:39.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:01:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:01:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:01:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:01:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:39.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2026-02-20T22:01:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_37 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:01:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:01:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_37 2026-02-20T22:01:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T22:01:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_37 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:01:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:01:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:39.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_37 2026-02-20T22:01:39.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T22:01:39.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:39.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:39.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_37 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:01:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:01:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:01:39.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_37 2026-02-20T22:01:39.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:01:39.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:01:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:01:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:01:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_37 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_37 2026-02-20T22:01:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_37 2026-02-20T22:01:40.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 158 2026-02-20T22:01:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_37 2026-02-20T22:01:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_37.status 2026-02-20T22:01:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_37.status 2026-02-20T22:01:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_37.status 2026-02-20T22:01:40.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_37.status 2026-02-20T22:01:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_37.status 2026-02-20T22:01:40.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.status 2026-02-20T22:01:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_37 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_37.status 2026-02-20T22:01:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_37.status 2026-02-20T22:01:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_37 --format xml 2026-02-20T22:01:40.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:40.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:01:40.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=158 2026-02-20T22:01:40.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:40.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:40.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:01:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.status 2026-02-20T22:01:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.status 2026-02-20T22:01:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-20T22:01:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=71 2026-02-20T22:01:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 158 = 71 2026-02-20T22:01:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:40.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:01:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.status 2026-02-20T22:01:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.status 2026-02-20T22:01:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-20T22:01:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:41.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=71 2026-02-20T22:01:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 158 = 71 2026-02-20T22:01:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:41.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.status 2026-02-20T22:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.status 2026-02-20T22:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-20T22:01:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:42.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:42.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:01:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:01:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:01:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:01:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 158 = '' 2026-02-20T22:01:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:01:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_37 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.status 2026-02-20T22:01:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.status 2026-02-20T22:01:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_37 --format xml 2026-02-20T22:01:43.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:43.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:43.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=158 2026-02-20T22:01:43.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 158 = 158 2026-02-20T22:01:43.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2026-02-20T22:01:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:01:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2026-02-20T22:01:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_37': 2026-02-20T22:01:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:01:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:01:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:01:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0579d6eff 2026-02-20T22:01:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0579d6eff 2026-02-20T22:01:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:01:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:01:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:01:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:01:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:53:55 2026 2026-02-20T22:01:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:53:55 2026 2026-02-20T22:01:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:53:55 2026 2026-02-20T22:01:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:01:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:01:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:01:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 433c55c5-23ec-4be7-a8a5-59991ae7abd8 2026-02-20T22:01:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:01:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_37 snap 2026-02-20T22:01:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:01:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:01:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_37.export 2026-02-20T22:01:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.export 2026-02-20T22:01:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_37.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.export 2026-02-20T22:01:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_37.export 2026-02-20T22:01:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.export 2026-02-20T22:01:45.158 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-20T22:01:45.158 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:01:45.159 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-20T22:01:45.159 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T22:01:45.159 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T22:01:45.161 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T22:01:45.161 INFO:tasks.rbd_mirror_thrash:waiting for 109.51460719031735 secs before thrashing 2026-02-20T22:01:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:45.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_37.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.export 2026-02-20T22:01:45.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_37.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_37.export 2026-02-20T22:01:45.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:01:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_38 2026-02-20T22:01:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_38 snap 2026-02-20T22:01:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:45.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:01:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2026-02-20T22:01:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2026-02-20T22:01:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:01:46.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:46.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:46.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2026-02-20T22:01:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:01:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:01:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:01:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:01:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:01:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2026-02-20T22:01:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:01:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:47.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:01:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:01:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:01:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:01:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:01:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:01:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:01:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:01:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:01:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:01:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:01:47.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:47.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2026-02-20T22:01:47.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-20T22:01:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:47.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:47.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:47.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_38 2026-02-20T22:01:47.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:01:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:01:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:01:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:47.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:01:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:01:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:47.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:01:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:01:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:01:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:01:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:47.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:47.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:01:47.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:01:47.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:01:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_38 2026-02-20T22:01:47.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-20T22:01:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:47.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_38 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:01:47.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:01:47.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:01:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_38 2026-02-20T22:01:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_38'] 2026-02-20T22:01:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:47.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_38 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:47.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:01:47.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:01:47.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:47.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:47.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:47.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_38 2026-02-20T22:01:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:01:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:01:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:01:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:01:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_38 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_38 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_38 2026-02-20T22:01:48.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 160 2026-02-20T22:01:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_38 2026-02-20T22:01:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_38.status 2026-02-20T22:01:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_38.status 2026-02-20T22:01:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:48.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_38.status 2026-02-20T22:01:48.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_38.status 2026-02-20T22:01:48.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_38.status 2026-02-20T22:01:48.671 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.status 2026-02-20T22:01:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_38 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_38.status 2026-02-20T22:01:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_38.status 2026-02-20T22:01:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_38 --format xml 2026-02-20T22:01:48.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:48.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:01:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=160 2026-02-20T22:01:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:01:48.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.status 2026-02-20T22:01:48.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:48.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:48.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:48.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.status 2026-02-20T22:01:48.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-20T22:01:48.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:48.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=72 2026-02-20T22:01:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 160 = 72 2026-02-20T22:01:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:01:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.status 2026-02-20T22:01:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.status 2026-02-20T22:01:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-20T22:01:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:49.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:49.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=72 2026-02-20T22:01:49.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 160 = 72 2026-02-20T22:01:49.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:49.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:01:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.status 2026-02-20T22:01:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:50.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.status 2026-02-20T22:01:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-20T22:01:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:50.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:01:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:01:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:01:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:01:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 160 = '' 2026-02-20T22:01:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:01:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_38 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.status 2026-02-20T22:01:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.status 2026-02-20T22:01:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_38 --format xml 2026-02-20T22:01:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=160 2026-02-20T22:01:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 160 = 160 2026-02-20T22:01:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2026-02-20T22:01:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:01:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2026-02-20T22:01:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_38': 2026-02-20T22:01:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:01:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:01:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:01:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b03fea2831 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b03fea2831 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:06 2026 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:06 2026 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:06 2026 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:01:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:01:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: f0d4e737-438c-4247-8539-7b6130b2be4d 2026-02-20T22:01:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:01:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_38 snap 2026-02-20T22:01:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:01:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:01:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_38.export 2026-02-20T22:01:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.export 2026-02-20T22:01:52.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_38.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.export 2026-02-20T22:01:52.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_38.export 2026-02-20T22:01:53.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:53.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.export 2026-02-20T22:01:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:01:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_38.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.export 2026-02-20T22:01:53.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_38.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_38.export 2026-02-20T22:01:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:01:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_39 2026-02-20T22:01:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_39 snap 2026-02-20T22:01:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:53.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:53.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:01:54.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2026-02-20T22:01:54.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:54.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:54.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:54.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2026-02-20T22:01:54.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:54.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:01:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:54.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:01:55.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2026-02-20T22:01:55.673 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:55.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:55.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:55.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:01:55.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:01:55.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:01:55.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2026-02-20T22:01:55.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:01:55.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:01:55.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:55.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:01:55.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:01:55.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:01:55.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:01:55.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:01:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:55.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:55.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:01:55.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2026-02-20T22:01:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T22:01:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:55.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:55.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:01:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:01:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:01:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:01:55.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:01:55.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:55.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:55.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:01:55.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2026-02-20T22:01:55.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T22:01:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:55.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_39 2026-02-20T22:01:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:01:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:01:55.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:55.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:01:55.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:01:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:55.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:01:55.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_39 2026-02-20T22:01:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T22:01:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:01:55.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:01:55.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_39 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:01:55.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:01:55.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:01:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:01:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:01:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:01:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:01:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:01:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:01:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:01:55.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:01:55.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:55.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:01:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_39 2026-02-20T22:01:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:01:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:01:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:01:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:01:55.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:01:55.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:01:55.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_39 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_39 2026-02-20T22:01:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:55.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:55.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:55.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_39 2026-02-20T22:01:56.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 162 2026-02-20T22:01:56.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_39 2026-02-20T22:01:56.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:01:56.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:56.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:01:56.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:01:56.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:56.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_39.status 2026-02-20T22:01:56.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_39.status 2026-02-20T22:01:56.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:56.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_39.status 2026-02-20T22:01:56.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_39.status 2026-02-20T22:01:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_39.status 2026-02-20T22:01:56.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:01:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.status 2026-02-20T22:01:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_39 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_39.status 2026-02-20T22:01:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:01:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:56.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_39.status 2026-02-20T22:01:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_39 --format xml 2026-02-20T22:01:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:56.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:01:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=162 2026-02-20T22:01:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:56.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:01:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.status 2026-02-20T22:01:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:56.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.status 2026-02-20T22:01:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-20T22:01:56.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:56.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=73 2026-02-20T22:01:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 162 = 73 2026-02-20T22:01:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:56.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:01:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.status 2026-02-20T22:01:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.status 2026-02-20T22:01:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-20T22:01:57.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:57.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=73 2026-02-20T22:01:57.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 162 = 73 2026-02-20T22:01:57.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:57.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:01:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.status 2026-02-20T22:01:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.status 2026-02-20T22:01:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-20T22:01:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:01:58.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:01:58.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:01:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:01:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:01:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 162 = '' 2026-02-20T22:01:58.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:01:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:01:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_39 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.status 2026-02-20T22:01:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.status 2026-02-20T22:01:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_39 --format xml 2026-02-20T22:01:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:01:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:01:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=162 2026-02-20T22:01:59.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 162 = 162 2026-02-20T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2026-02-20T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:01:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:00.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2026-02-20T22:02:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_39': 2026-02-20T22:02:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:02:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:02:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:02:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b08731d1ce 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b08731d1ce 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:17 2026 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:17 2026 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:17 2026 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:02:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:02:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:02:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 876e1da9-8a7e-447b-81f0-6c8df3b23c0c 2026-02-20T22:02:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:02:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_39 snap 2026-02-20T22:02:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:02:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:02:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_39.export 2026-02-20T22:02:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.export 2026-02-20T22:02:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_39.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.export 2026-02-20T22:02:00.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_39.export 2026-02-20T22:02:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:01.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.export 2026-02-20T22:02:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:01.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_39.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.export 2026-02-20T22:02:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_39.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_39.export 2026-02-20T22:02:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:02:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_40 2026-02-20T22:02:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_40 snap 2026-02-20T22:02:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:01.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:01.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:02:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2026-02-20T22:02:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2026-02-20T22:02:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:02.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:02.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:02:02.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:02.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:02.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2026-02-20T22:02:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:02:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:02:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:02:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:02:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:02:03.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:03.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:03.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:03.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2026-02-20T22:02:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T22:02:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:03.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2026-02-20T22:02:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:02:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:02:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:02:03.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2026-02-20T22:02:03.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T22:02:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_40 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:02:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:02:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:02:03.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:03.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_40 2026-02-20T22:02:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T22:02:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:03.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_40 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:03.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:02:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:02:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:02:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:02:03.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:02:03.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:02:03.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:03.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:03.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:03.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_40 2026-02-20T22:02:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:02:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:02:03.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:02:03.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:02:03.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:02:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:02:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:02:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_40 2026-02-20T22:02:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:03.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_40 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:03.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_40 2026-02-20T22:02:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 164 2026-02-20T22:02:04.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_40 2026-02-20T22:02:04.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:04.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:04.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:04.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:04.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:04.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_40.status 2026-02-20T22:02:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_40.status 2026-02-20T22:02:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:04.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_40.status 2026-02-20T22:02:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_40.status 2026-02-20T22:02:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_40.status 2026-02-20T22:02:04.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:04.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.status 2026-02-20T22:02:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_40 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_40.status 2026-02-20T22:02:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_40.status 2026-02-20T22:02:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_40 --format xml 2026-02-20T22:02:04.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:02:04.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=164 2026-02-20T22:02:04.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:04.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:02:04.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.status 2026-02-20T22:02:04.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:04.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:04.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:04.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.status 2026-02-20T22:02:04.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-20T22:02:04.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=74 2026-02-20T22:02:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 164 = 74 2026-02-20T22:02:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:02:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.status 2026-02-20T22:02:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.status 2026-02-20T22:02:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-20T22:02:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:05.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=74 2026-02-20T22:02:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 164 = 74 2026-02-20T22:02:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:02:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.status 2026-02-20T22:02:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:06.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.status 2026-02-20T22:02:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-20T22:02:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:06.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:02:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:02:06.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:02:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:02:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 164 = '' 2026-02-20T22:02:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:06.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:02:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_40 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.status 2026-02-20T22:02:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.status 2026-02-20T22:02:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_40 --format xml 2026-02-20T22:02:07.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=164 2026-02-20T22:02:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 164 = 164 2026-02-20T22:02:07.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2026-02-20T22:02:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:02:07.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:08.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2026-02-20T22:02:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_40': 2026-02-20T22:02:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:02:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:02:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:02:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b01c5eef37 2026-02-20T22:02:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b01c5eef37 2026-02-20T22:02:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:32 2026 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:32 2026 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:32 2026 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:02:08.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: c4c8b7cd-d483-42c9-9489-fa930aaeae6b 2026-02-20T22:02:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:02:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_40 snap 2026-02-20T22:02:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:02:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:02:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_40.export 2026-02-20T22:02:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.export 2026-02-20T22:02:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_40.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.export 2026-02-20T22:02:08.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_40.export 2026-02-20T22:02:09.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.export 2026-02-20T22:02:09.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:09.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_40.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.export 2026-02-20T22:02:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_40.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_40.export 2026-02-20T22:02:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:02:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_41 2026-02-20T22:02:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_41 snap 2026-02-20T22:02:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:02:10.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2026-02-20T22:02:10.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:10.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:10.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2026-02-20T22:02:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:02:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2026-02-20T22:02:11.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:02:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:02:11.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:02:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2026-02-20T22:02:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:02:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:02:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:02:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:02:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:02:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:11.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:11.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:02:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:02:11.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:02:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:11.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2026-02-20T22:02:11.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T22:02:11.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:11.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2026-02-20T22:02:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:02:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:02:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:02:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:02:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:11.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2026-02-20T22:02:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T22:02:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_41 2026-02-20T22:02:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:02:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:02:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:02:11.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:02:11.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:11.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_41 2026-02-20T22:02:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T22:02:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:11.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_41 2026-02-20T22:02:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:02:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:02:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:02:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:02:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:02:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:11.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_41 2026-02-20T22:02:11.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:02:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:02:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:02:11.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:02:11.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:02:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:02:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:02:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_41 2026-02-20T22:02:11.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_41 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:11.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_41 2026-02-20T22:02:12.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 166 2026-02-20T22:02:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_41 2026-02-20T22:02:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:12.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_41.status 2026-02-20T22:02:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_41.status 2026-02-20T22:02:12.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:12.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_41.status 2026-02-20T22:02:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_41.status 2026-02-20T22:02:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_41.status 2026-02-20T22:02:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.status 2026-02-20T22:02:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_41 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_41.status 2026-02-20T22:02:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_41.status 2026-02-20T22:02:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_41 --format xml 2026-02-20T22:02:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:12.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:02:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=166 2026-02-20T22:02:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:02:12.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.status 2026-02-20T22:02:12.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:12.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:12.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:12.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.status 2026-02-20T22:02:12.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-20T22:02:12.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=75 2026-02-20T22:02:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 166 = 75 2026-02-20T22:02:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:02:13.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.status 2026-02-20T22:02:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.status 2026-02-20T22:02:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-20T22:02:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:13.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=75 2026-02-20T22:02:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 166 = 75 2026-02-20T22:02:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:13.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:02:14.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.status 2026-02-20T22:02:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.status 2026-02-20T22:02:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-20T22:02:14.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:14.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:14.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:02:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:02:14.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:02:14.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:02:14.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 166 = '' 2026-02-20T22:02:14.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:14.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:02:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_41 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.status 2026-02-20T22:02:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.status 2026-02-20T22:02:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_41 --format xml 2026-02-20T22:02:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=166 2026-02-20T22:02:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 166 = 166 2026-02-20T22:02:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2026-02-20T22:02:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:02:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2026-02-20T22:02:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_41': 2026-02-20T22:02:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:02:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0448cd6aa 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0448cd6aa 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:43 2026 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:43 2026 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:43 2026 2026-02-20T22:02:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:02:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:02:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:02:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: f3be01dc-d6c3-44a9-9f77-d87df8a565ca 2026-02-20T22:02:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:02:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_41 snap 2026-02-20T22:02:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:02:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:02:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_41.export 2026-02-20T22:02:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.export 2026-02-20T22:02:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_41.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.export 2026-02-20T22:02:16.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_41.export 2026-02-20T22:02:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:17.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.export 2026-02-20T22:02:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_41.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.export 2026-02-20T22:02:17.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_41.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_41.export 2026-02-20T22:02:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:02:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_42 2026-02-20T22:02:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_42 snap 2026-02-20T22:02:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:19.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2026-02-20T22:02:19.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:19.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:19.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:19.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:02:19.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:02:19.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:02:19.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2026-02-20T22:02:19.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:02:19.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:02:19.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:19.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:02:19.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:02:19.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:02:19.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:19.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:02:19.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:02:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:19.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:19.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:02:19.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:02:19.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:02:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:19.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2026-02-20T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-20T22:02:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:19.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_42 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:19.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:02:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:02:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:02:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_42 2026-02-20T22:02:19.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-20T22:02:19.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:19.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_42 2026-02-20T22:02:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:02:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:02:19.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:19.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:02:19.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:02:19.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:02:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:19.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_42 2026-02-20T22:02:19.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_42'] 2026-02-20T22:02:19.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:19.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_42 2026-02-20T22:02:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:02:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:19.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:02:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:02:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:02:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:19.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:02:19.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:02:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_42 2026-02-20T22:02:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:02:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:02:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:02:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:02:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:02:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:02:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:02:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_42 2026-02-20T22:02:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:02:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:02:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_42 2026-02-20T22:02:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_42 2026-02-20T22:02:20.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 168 2026-02-20T22:02:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_42 2026-02-20T22:02:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_42.status 2026-02-20T22:02:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_42.status 2026-02-20T22:02:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_42.status 2026-02-20T22:02:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_42.status 2026-02-20T22:02:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_42.status 2026-02-20T22:02:20.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:20.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.status 2026-02-20T22:02:20.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_42 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_42.status 2026-02-20T22:02:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_42.status 2026-02-20T22:02:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_42 --format xml 2026-02-20T22:02:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:20.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=168 2026-02-20T22:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:02:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.status 2026-02-20T22:02:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.status 2026-02-20T22:02:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-20T22:02:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:20.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:20.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=76 2026-02-20T22:02:20.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 168 = 76 2026-02-20T22:02:20.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:20.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:02:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.status 2026-02-20T22:02:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.status 2026-02-20T22:02:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-20T22:02:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:21.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=76 2026-02-20T22:02:21.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 168 = 76 2026-02-20T22:02:21.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:21.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.status 2026-02-20T22:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.status 2026-02-20T22:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-20T22:02:22.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:22.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:02:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:02:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:02:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:02:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 168 = '' 2026-02-20T22:02:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:02:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_42 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.status 2026-02-20T22:02:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.status 2026-02-20T22:02:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_42 --format xml 2026-02-20T22:02:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:23.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=168 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 168 = 168 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:02:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2026-02-20T22:02:24.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_42': 2026-02-20T22:02:24.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:02:24.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:02:24.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:02:24.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b07b15a2b8 2026-02-20T22:02:24.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b07b15a2b8 2026-02-20T22:02:24.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:02:24.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:50 2026 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:50 2026 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:50 2026 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: e3fc513d-202f-403b-8d2f-5ae273783233 2026-02-20T22:02:24.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:02:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_42 snap 2026-02-20T22:02:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:02:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:02:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_42.export 2026-02-20T22:02:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.export 2026-02-20T22:02:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_42.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.export 2026-02-20T22:02:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_42.export 2026-02-20T22:02:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:25.089 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.export 2026-02-20T22:02:25.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_42.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.export 2026-02-20T22:02:25.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_42.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_42.export 2026-02-20T22:02:25.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:02:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_43 2026-02-20T22:02:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_43 snap 2026-02-20T22:02:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:25.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:02:26.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2026-02-20T22:02:26.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:26.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2026-02-20T22:02:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:02:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:26.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2026-02-20T22:02:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:02:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:02:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:02:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2026-02-20T22:02:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:02:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:02:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:02:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:02:27.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:02:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:02:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:02:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:02:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:02:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:27.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:27.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:27.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:27.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:27.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:02:27.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:02:27.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:02:27.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:27.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:27.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:27.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-20T22:02:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T22:02:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:27.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:27.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:27.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-20T22:02:27.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:27.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:27.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:02:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:02:27.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:02:27.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:27.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:27.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:27.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-20T22:02:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T22:02:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:27.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:27.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:02:27.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:02:27.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:02:27.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:27.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-20T22:02:27.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T22:02:27.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:27.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:27.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_43 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:27.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:02:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:02:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:02:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_43 2026-02-20T22:02:28.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:02:28.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:02:28.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:02:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:02:28.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:02:28.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:02:28.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:02:28.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:28.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_43 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_43 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_43 2026-02-20T22:02:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 170 2026-02-20T22:02:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_43 2026-02-20T22:02:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_43.status 2026-02-20T22:02:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_43.status 2026-02-20T22:02:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_43.status 2026-02-20T22:02:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_43.status 2026-02-20T22:02:28.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_43.status 2026-02-20T22:02:28.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:28.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.status 2026-02-20T22:02:28.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_43 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_43.status 2026-02-20T22:02:28.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:28.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:28.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:28.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_43.status 2026-02-20T22:02:28.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_43 --format xml 2026-02-20T22:02:28.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:28.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:02:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=170 2026-02-20T22:02:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:02:28.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.status 2026-02-20T22:02:28.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:28.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:28.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:28.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.status 2026-02-20T22:02:28.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-20T22:02:28.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:28.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=77 2026-02-20T22:02:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 170 = 77 2026-02-20T22:02:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:02:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.status 2026-02-20T22:02:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.status 2026-02-20T22:02:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-20T22:02:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:29.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:29.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=77 2026-02-20T22:02:29.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 170 = 77 2026-02-20T22:02:29.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:29.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:02:30.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.status 2026-02-20T22:02:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.status 2026-02-20T22:02:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-20T22:02:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:30.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:30.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:02:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:02:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:02:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:02:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 170 = '' 2026-02-20T22:02:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:02:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_43 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.status 2026-02-20T22:02:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.status 2026-02-20T22:02:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_43 --format xml 2026-02-20T22:02:31.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:31.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=170 2026-02-20T22:02:31.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 170 = 170 2026-02-20T22:02:31.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:31.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2026-02-20T22:02:31.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:31.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:31.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:31.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:31.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:31.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:02:31.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2026-02-20T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_43': 2026-02-20T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b09761bdc9 2026-02-20T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b09761bdc9 2026-02-20T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:54:58 2026 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:54:58 2026 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:54:58 2026 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: b3411393-1507-494e-b3af-7d1d3d485652 2026-02-20T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:02:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_43 snap 2026-02-20T22:02:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:02:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:02:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_43.export 2026-02-20T22:02:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.export 2026-02-20T22:02:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_43.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.export 2026-02-20T22:02:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_43.export 2026-02-20T22:02:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:33.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.export 2026-02-20T22:02:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:33.275 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_43.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.export 2026-02-20T22:02:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_43.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_43.export 2026-02-20T22:02:33.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:33.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:02:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_44 2026-02-20T22:02:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_44 snap 2026-02-20T22:02:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:02:34.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2026-02-20T22:02:34.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2026-02-20T22:02:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:02:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:02:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:02:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2026-02-20T22:02:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:02:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:02:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:02:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:02:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:02:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:02:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:35.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:02:35.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:02:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:02:35.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:35.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2026-02-20T22:02:35.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T22:02:35.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:35.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_44 2026-02-20T22:02:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:02:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:02:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:02:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:35.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:35.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:02:35.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:02:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_44 2026-02-20T22:02:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T22:02:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_44 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:02:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:02:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:35.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_44 2026-02-20T22:02:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T22:02:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_44 2026-02-20T22:02:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:02:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:02:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:02:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:02:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:02:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_44 2026-02-20T22:02:36.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:02:36.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:02:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:02:36.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:02:36.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:02:36.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:02:36.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:02:36.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:36.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_44 2026-02-20T22:02:36.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:36.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_44 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:36.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_44 2026-02-20T22:02:36.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 172 2026-02-20T22:02:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_44 2026-02-20T22:02:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_44.status 2026-02-20T22:02:36.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_44.status 2026-02-20T22:02:36.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:36.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_44.status 2026-02-20T22:02:36.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_44.status 2026-02-20T22:02:36.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_44.status 2026-02-20T22:02:36.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.status 2026-02-20T22:02:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_44 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_44.status 2026-02-20T22:02:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_44.status 2026-02-20T22:02:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_44 --format xml 2026-02-20T22:02:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:02:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=172 2026-02-20T22:02:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:02:36.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.status 2026-02-20T22:02:36.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:36.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:36.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:36.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.status 2026-02-20T22:02:36.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-20T22:02:36.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:37.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=78 2026-02-20T22:02:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 172 = 78 2026-02-20T22:02:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:02:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.status 2026-02-20T22:02:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.status 2026-02-20T22:02:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-20T22:02:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=78 2026-02-20T22:02:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 172 = 78 2026-02-20T22:02:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:02:38.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.status 2026-02-20T22:02:38.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:38.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:38.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:38.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.status 2026-02-20T22:02:38.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-20T22:02:38.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:38.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:02:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:02:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:02:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:02:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 172 = '' 2026-02-20T22:02:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_44 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.status 2026-02-20T22:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.status 2026-02-20T22:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_44 --format xml 2026-02-20T22:02:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=172 2026-02-20T22:02:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 172 = 172 2026-02-20T22:02:39.926 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2026-02-20T22:02:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:02:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2026-02-20T22:02:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_44': 2026-02-20T22:02:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:02:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:02:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:02:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0746c9fe2 2026-02-20T22:02:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0746c9fe2 2026-02-20T22:02:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:02:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:09 2026 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:09 2026 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:09 2026 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 9c7289c8-3641-48c4-aeec-d1a3caf6cc21 2026-02-20T22:02:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:02:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_44 snap 2026-02-20T22:02:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:02:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:02:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_44.export 2026-02-20T22:02:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.export 2026-02-20T22:02:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_44.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.export 2026-02-20T22:02:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_44.export 2026-02-20T22:02:41.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.export 2026-02-20T22:02:41.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_44.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.export 2026-02-20T22:02:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_44.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_44.export 2026-02-20T22:02:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:02:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_45 2026-02-20T22:02:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_45 snap 2026-02-20T22:02:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:02:42.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2026-02-20T22:02:42.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:42.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2026-02-20T22:02:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:02:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:42.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:43.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2026-02-20T22:02:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:02:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:02:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:02:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2026-02-20T22:02:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:02:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:02:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:02:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:02:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:02:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:02:43.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:02:43.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:02:43.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:43.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2026-02-20T22:02:43.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:02:43.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:43.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:43.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:43.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:43.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:02:43.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:02:43.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:02:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2026-02-20T22:02:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T22:02:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_45 2026-02-20T22:02:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:02:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:02:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:02:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:02:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:43.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_45 2026-02-20T22:02:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T22:02:43.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:43.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_45 2026-02-20T22:02:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:02:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:02:43.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:02:43.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:02:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:02:43.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:02:43.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:43.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_45 2026-02-20T22:02:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T22:02:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:43.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_45 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:02:43.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:02:43.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:43.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:43.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:43.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_45 2026-02-20T22:02:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:02:44.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:02:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:02:44.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:02:44.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:02:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:02:44.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_45 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:02:44.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_45 2026-02-20T22:02:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:44.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_45 2026-02-20T22:02:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 174 2026-02-20T22:02:44.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_45 2026-02-20T22:02:44.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:44.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:44.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:44.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:44.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:44.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_45.status 2026-02-20T22:02:44.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_45.status 2026-02-20T22:02:44.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:44.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_45.status 2026-02-20T22:02:44.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_45.status 2026-02-20T22:02:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_45.status 2026-02-20T22:02:44.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:44.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.status 2026-02-20T22:02:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_45 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_45.status 2026-02-20T22:02:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_45.status 2026-02-20T22:02:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_45 --format xml 2026-02-20T22:02:44.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:02:44.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=174 2026-02-20T22:02:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:44.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:02:44.980 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.status 2026-02-20T22:02:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.status 2026-02-20T22:02:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-20T22:02:44.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:45.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=79 2026-02-20T22:02:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 174 = 79 2026-02-20T22:02:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:02:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.status 2026-02-20T22:02:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.status 2026-02-20T22:02:45.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-20T22:02:45.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:45.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:45.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=79 2026-02-20T22:02:45.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 174 = 79 2026-02-20T22:02:45.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:45.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:02:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.status 2026-02-20T22:02:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.status 2026-02-20T22:02:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-20T22:02:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:46.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:02:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:02:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:02:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:02:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 174 = '' 2026-02-20T22:02:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:46.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:02:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_45 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.status 2026-02-20T22:02:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.status 2026-02-20T22:02:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_45 --format xml 2026-02-20T22:02:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:47.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=174 2026-02-20T22:02:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 174 = 174 2026-02-20T22:02:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2026-02-20T22:02:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:47.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:02:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:48.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_45': 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0ff2172a2 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0ff2172a2 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:02:49.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:02:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:19 2026 2026-02-20T22:02:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:19 2026 2026-02-20T22:02:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:19 2026 2026-02-20T22:02:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:02:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:02:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:02:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: cf134cc1-67a2-4edd-85cf-ecb315b92e21 2026-02-20T22:02:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:02:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_45 snap 2026-02-20T22:02:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:02:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:02:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_45.export 2026-02-20T22:02:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.export 2026-02-20T22:02:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_45.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.export 2026-02-20T22:02:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_45.export 2026-02-20T22:02:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:49.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.export 2026-02-20T22:02:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_45.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.export 2026-02-20T22:02:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_45.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_45.export 2026-02-20T22:02:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:02:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_46 2026-02-20T22:02:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_46 snap 2026-02-20T22:02:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:49.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:02:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2026-02-20T22:02:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2026-02-20T22:02:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:02:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2026-02-20T22:02:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:02:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:02:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:02:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-20T22:02:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:02:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:02:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:02:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:02:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:02:51.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:02:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-20T22:02:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T22:02:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-20T22:02:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:02:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:02:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:02:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-20T22:02:51.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T22:02:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:51.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:51.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:51.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:51.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:02:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:02:51.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:02:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-02-20T22:02:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T22:02:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_46 2026-02-20T22:02:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:51.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:51.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:51.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:51.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:02:51.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:02:51.963 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:02:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_46 2026-02-20T22:02:52.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:02:52.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:02:52.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:02:52.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:02:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:02:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:02:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:02:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_46 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_46 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:52.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:52.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_46 2026-02-20T22:02:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 176 2026-02-20T22:02:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_46 2026-02-20T22:02:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:02:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:02:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:02:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:52.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_46.status 2026-02-20T22:02:52.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_46.status 2026-02-20T22:02:52.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:52.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_46.status 2026-02-20T22:02:52.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_46.status 2026-02-20T22:02:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_46.status 2026-02-20T22:02:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:02:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.status 2026-02-20T22:02:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_46 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_46.status 2026-02-20T22:02:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_46.status 2026-02-20T22:02:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_46 --format xml 2026-02-20T22:02:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:52.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:02:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=176 2026-02-20T22:02:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:52.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:02:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.status 2026-02-20T22:02:52.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:52.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:52.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:52.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.status 2026-02-20T22:02:52.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-20T22:02:52.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:53.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=80 2026-02-20T22:02:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 176 = 80 2026-02-20T22:02:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:53.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:02:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.status 2026-02-20T22:02:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.status 2026-02-20T22:02:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-20T22:02:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=80 2026-02-20T22:02:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 176 = 80 2026-02-20T22:02:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:02:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.status 2026-02-20T22:02:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.status 2026-02-20T22:02:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-20T22:02:54.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:54.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:02:54.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:02:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:02:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:02:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:02:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 176 = '' 2026-02-20T22:02:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:02:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:02:55.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_46 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.status 2026-02-20T22:02:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.status 2026-02-20T22:02:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_46 --format xml 2026-02-20T22:02:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:02:55.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:02:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=176 2026-02-20T22:02:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 176 = 176 2026-02-20T22:02:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2026-02-20T22:02:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:02:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:56.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2026-02-20T22:02:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_46': 2026-02-20T22:02:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:02:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0f279e407 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0f279e407 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:30 2026 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:30 2026 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:30 2026 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:02:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:02:56.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:02:56.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 8e7ce785-f9ba-4268-a043-f8b326632069 2026-02-20T22:02:56.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:02:56.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_46 snap 2026-02-20T22:02:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:02:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:02:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_46.export 2026-02-20T22:02:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.export 2026-02-20T22:02:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_46.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.export 2026-02-20T22:02:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_46.export 2026-02-20T22:02:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:57.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.export 2026-02-20T22:02:57.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:02:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_46.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.export 2026-02-20T22:02:57.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_46.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_46.export 2026-02-20T22:02:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:02:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:02:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_47 2026-02-20T22:02:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_47 snap 2026-02-20T22:02:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:02:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:02:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:02:57.426 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:02:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2026-02-20T22:02:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:02:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2026-02-20T22:02:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:58.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:02:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2026-02-20T22:02:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:02:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:02:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:02:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:02:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:02:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:02:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2026-02-20T22:02:59.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:02:59.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:02:59.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:59.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:02:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:02:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:02:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:02:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:02:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:02:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:02:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2026-02-20T22:02:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:02:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:02:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:59.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:59.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:02:59.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:02:59.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:02:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:02:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2026-02-20T22:02:59.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-20T22:02:59.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:59.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_47 2026-02-20T22:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:59.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:02:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:02:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:02:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_47 2026-02-20T22:02:59.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-20T22:02:59.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:59.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:59.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_47 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:02:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:59.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:02:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:02:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:02:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:02:59.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:02:59.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:02:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_47 2026-02-20T22:02:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-20T22:02:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:02:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:02:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:02:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_47 2026-02-20T22:02:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:02:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:02:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:02:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:02:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:02:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:02:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:02:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:02:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:02:59.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_47 2026-02-20T22:03:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:03:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:03:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:03:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:03:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:03:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:03:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:03:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_47 2026-02-20T22:03:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_47 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_47 2026-02-20T22:03:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 178 2026-02-20T22:03:00.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_47 2026-02-20T22:03:00.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:03:00.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:00.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:03:00.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:03:00.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_47.status 2026-02-20T22:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_47.status 2026-02-20T22:03:00.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:03:00.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_47.status 2026-02-20T22:03:00.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_47.status 2026-02-20T22:03:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_47.status 2026-02-20T22:03:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:03:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.status 2026-02-20T22:03:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_47 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_47.status 2026-02-20T22:03:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_47.status 2026-02-20T22:03:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_47 --format xml 2026-02-20T22:03:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:00.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:03:00.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=178 2026-02-20T22:03:00.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:03:00.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:00.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:03:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.status 2026-02-20T22:03:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.status 2026-02-20T22:03:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-20T22:03:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=81 2026-02-20T22:03:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 178 = 81 2026-02-20T22:03:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:03:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.status 2026-02-20T22:03:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:01.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.status 2026-02-20T22:03:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-20T22:03:01.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=81 2026-02-20T22:03:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 178 = 81 2026-02-20T22:03:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:03:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.status 2026-02-20T22:03:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.status 2026-02-20T22:03:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-20T22:03:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:02.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:03:02.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:02.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:03:02.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:03:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:03:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:03:02.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 178 = '' 2026-02-20T22:03:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:02.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:03:03.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_47 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.status 2026-02-20T22:03:03.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:03.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:03.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:03.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.status 2026-02-20T22:03:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_47 --format xml 2026-02-20T22:03:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:03.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:03.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=178 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 178 = 178 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:03:03.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:03:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2026-02-20T22:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_47': 2026-02-20T22:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b093364608 2026-02-20T22:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b093364608 2026-02-20T22:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:03:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:38 2026 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:38 2026 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:38 2026 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 740d48a2-b3be-43ea-a507-e1c8ff34cb29 2026-02-20T22:03:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:03:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_47 snap 2026-02-20T22:03:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:03:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:03:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_47.export 2026-02-20T22:03:04.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.export 2026-02-20T22:03:04.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_47.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.export 2026-02-20T22:03:04.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_47.export 2026-02-20T22:03:05.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:03:05.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.export 2026-02-20T22:03:05.324 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:03:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_47.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.export 2026-02-20T22:03:05.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_47.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_47.export 2026-02-20T22:03:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_48 2026-02-20T22:03:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_48 snap 2026-02-20T22:03:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:05.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:03:06.592 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7896021122352943) > weight (0.25) 2026-02-20T22:03:06.593 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4786822887566837) > weight (0.25) 2026-02-20T22:03:06.593 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.3149690634226412) > weight (0.25) 2026-02-20T22:03:06.593 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-20T22:03:06.593 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-20T22:03:06.593 INFO:tasks.rbd_mirror_thrash:waiting for 6.468010786187808 secs before reviving daemons 2026-02-20T22:03:06.594 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:03:06.589+0000 7fb00881f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 19474) UID: 1000 2026-02-20T22:03:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2026-02-20T22:03:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2026-02-20T22:03:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:03:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:03:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:03:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:03:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2026-02-20T22:03:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:07.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:03:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:03:07.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:03:07.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:03:07.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:03:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:03:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:03:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:03:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:03:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:03:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:03:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:03:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:03:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2026-02-20T22:03:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T22:03:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:03:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:03:07.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_48 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:03:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:03:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:03:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:03:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:03:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:03:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:03:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_48 2026-02-20T22:03:07.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T22:03:07.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:03:07.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:03:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_48 2026-02-20T22:03:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:03:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:03:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:03:07.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:03:07.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:03:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:03:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:03:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:03:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_48 2026-02-20T22:03:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T22:03:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:03:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:03:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_48 2026-02-20T22:03:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:03:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:03:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:03:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:07.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:07.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:03:07.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:03:07.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:03:07.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:03:07.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:03:07.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_48 2026-02-20T22:03:08.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:03:08.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:03:08.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:03:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:03:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:03:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:03:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:03:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_48 2026-02-20T22:03:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:03:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:03:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:03:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:03:08.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:03:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_48 2026-02-20T22:03:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:08.071 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_48 2026-02-20T22:03:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 180 2026-02-20T22:03:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_48 2026-02-20T22:03:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:03:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:03:08.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:03:08.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:08.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_48.status 2026-02-20T22:03:08.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_48.status 2026-02-20T22:03:08.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:03:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_48.status 2026-02-20T22:03:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_48.status 2026-02-20T22:03:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_48.status 2026-02-20T22:03:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:03:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.status 2026-02-20T22:03:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_48 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_48.status 2026-02-20T22:03:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_48.status 2026-02-20T22:03:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_48 --format xml 2026-02-20T22:03:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:08.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:03:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=180 2026-02-20T22:03:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:03:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:08.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:03:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.status 2026-02-20T22:03:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.status 2026-02-20T22:03:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-20T22:03:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:09.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=82 2026-02-20T22:03:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 180 = 82 2026-02-20T22:03:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:09.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:03:09.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.status 2026-02-20T22:03:09.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:09.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:09.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:09.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.status 2026-02-20T22:03:09.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-20T22:03:09.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:03:09.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:03:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:03:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:03:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:03:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 180 = '' 2026-02-20T22:03:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:03:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.status 2026-02-20T22:03:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.status 2026-02-20T22:03:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-20T22:03:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:10.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:03:10.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:10.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:03:10.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:03:10.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:03:10.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:03:10.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 180 = '' 2026-02-20T22:03:10.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:10.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:03:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_48 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.status 2026-02-20T22:03:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:12.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.status 2026-02-20T22:03:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_48 --format xml 2026-02-20T22:03:12.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:12.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=180 2026-02-20T22:03:12.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 180 = 180 2026-02-20T22:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2026-02-20T22:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:03:12.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:03:12.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:03:13.062 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-20T22:03:13.062 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:03:13.062 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-20T22:03:13.062 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-20T22:03:13.062 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-20T22:03:13.064 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-20T22:03:13.065 INFO:tasks.rbd_mirror_thrash:waiting for 67.54774822432891 secs before thrashing 2026-02-20T22:03:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2026-02-20T22:03:13.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_48': 2026-02-20T22:03:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:03:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:03:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:03:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b01dcfe5cc 2026-02-20T22:03:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b01dcfe5cc 2026-02-20T22:03:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:03:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:03:13.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:03:13.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:03:13.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:55:49 2026 2026-02-20T22:03:13.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:55:49 2026 2026-02-20T22:03:13.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:55:49 2026 2026-02-20T22:03:13.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:03:13.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:03:13.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:03:13.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 4a44519a-03e3-4a42-bf80-6ed4a8037a65 2026-02-20T22:03:13.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:03:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_48 snap 2026-02-20T22:03:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:03:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:03:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_48.export 2026-02-20T22:03:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.export 2026-02-20T22:03:13.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_48.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.export 2026-02-20T22:03:13.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_48.export 2026-02-20T22:03:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:03:13.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.export 2026-02-20T22:03:13.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:03:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_48.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.export 2026-02-20T22:03:13.587 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_48.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_48.export 2026-02-20T22:03:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_49 2026-02-20T22:03:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_49 snap 2026-02-20T22:03:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:13.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:03:14.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:03:14.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:03:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:03:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2026-02-20T22:03:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:03:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:03:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:03:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2026-02-20T22:03:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:03:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:03:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:03:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:03:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:03:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:03:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:03:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:03:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:03:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2026-02-20T22:03:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:03:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:03:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:03:15.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:03:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:03:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:03:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:03:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:03:15.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:03:15.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2026-02-20T22:03:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T22:03:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:03:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:03:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:03:15.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:03:15.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:15.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:15.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:03:15.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:03:15.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:03:15.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:03:15.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:03:15.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:03:15.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2026-02-20T22:03:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T22:03:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:03:15.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:03:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2026-02-20T22:03:15.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:15.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:03:15.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:03:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:03:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:03:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:03:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:03:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:03:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:03:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2026-02-20T22:03:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T22:03:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:03:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:03:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_49 2026-02-20T22:03:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:03:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:03:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:03:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:03:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:03:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:03:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:03:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:03:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_49 2026-02-20T22:03:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:03:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:03:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:03:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:03:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:03:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:03:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:03:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_49 2026-02-20T22:03:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_49 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_49 2026-02-20T22:03:16.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 182 2026-02-20T22:03:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_49 2026-02-20T22:03:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:03:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:03:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:03:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_49.status 2026-02-20T22:03:16.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_49.status 2026-02-20T22:03:16.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:03:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_49.status 2026-02-20T22:03:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_49.status 2026-02-20T22:03:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_49.status 2026-02-20T22:03:16.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:03:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.status 2026-02-20T22:03:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_49 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_49.status 2026-02-20T22:03:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_49.status 2026-02-20T22:03:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_49 --format xml 2026-02-20T22:03:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:16.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:03:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=182 2026-02-20T22:03:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:03:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:16.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:03:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.status 2026-02-20T22:03:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.status 2026-02-20T22:03:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-20T22:03:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:17.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=83 2026-02-20T22:03:17.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 182 = 83 2026-02-20T22:03:17.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:17.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:03:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.status 2026-02-20T22:03:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.status 2026-02-20T22:03:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-20T22:03:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=83 2026-02-20T22:03:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 182 = 83 2026-02-20T22:03:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:03:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.status 2026-02-20T22:03:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.status 2026-02-20T22:03:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-20T22:03:18.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:03:18.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:18.332 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:-:1.1: Document is empty 2026-02-20T22:03:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: 2026-02-20T22:03:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:^ 2026-02-20T22:03:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id= 2026-02-20T22:03:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 182 = '' 2026-02-20T22:03:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:18.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:03:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_49 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.status 2026-02-20T22:03:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.status 2026-02-20T22:03:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_49 --format xml 2026-02-20T22:03:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:19.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=182 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 182 = 182 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:03:19.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:03:20.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2026-02-20T22:03:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_49': 2026-02-20T22:03:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:03:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:03:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:03:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0472ca26e 2026-02-20T22:03:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0472ca26e 2026-02-20T22:03:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:03:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:03:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:03:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:03:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:56:03 2026 2026-02-20T22:03:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:56:03 2026 2026-02-20T22:03:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:56:03 2026 2026-02-20T22:03:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:03:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:03:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:03:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: d0428f4e-f589-457e-8185-f6b0bba89f12 2026-02-20T22:03:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:03:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_49 snap 2026-02-20T22:03:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:03:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:03:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_49.export 2026-02-20T22:03:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.export 2026-02-20T22:03:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_49.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.export 2026-02-20T22:03:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_49.export 2026-02-20T22:03:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:03:21.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.export 2026-02-20T22:03:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:03:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_49.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.export 2026-02-20T22:03:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_49.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_49.export 2026-02-20T22:03:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_50 2026-02-20T22:03:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_snap cluster2 mirror image_50 snap 2026-02-20T22:03:21.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:21.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T22:03:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2026-02-20T22:03:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2026-02-20T22:03:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=started 2026-02-20T22:03:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:03:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:03:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:03:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2026-02-20T22:03:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=started 2026-02-20T22:03:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_result 2026-02-20T22:03:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=stopped 2026-02-20T22:03:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2026-02-20T22:03:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_instance=cluster1 2026-02-20T22:03:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -i state 2026-02-20T22:03:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:03:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance=cluster1 2026-02-20T22:03:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local loop_instance 2026-02-20T22:03:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T22:03:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sleep 0 2026-02-20T22:03:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T22:03:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ seq 0 3 2026-02-20T22:03:23.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:0 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:0 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:03:23.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=0 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:0 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:0 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:23.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=0 2026-02-20T22:03:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T22:03:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=0 2026-02-20T22:03:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=0 2026-02-20T22:03:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T22:03:23.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:03:23.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:03:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T22:03:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2026-02-20T22:03:23.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T22:03:23.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:03:23.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:03:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2026-02-20T22:03:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:1 2026-02-20T22:03:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T22:03:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:1 2026-02-20T22:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=1 2026-02-20T22:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T22:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=1 2026-02-20T22:03:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:1 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:1 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=1 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T22:03:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=1 2026-02-20T22:03:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=1 2026-02-20T22:03:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T22:03:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:03:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:03:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T22:03:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2026-02-20T22:03:23.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T22:03:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:03:23.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:03:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_50 2026-02-20T22:03:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:2 2026-02-20T22:03:23.919 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:2 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=2 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=2 2026-02-20T22:03:23.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:2 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:2 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=2 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:23.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=2 2026-02-20T22:03:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=2 2026-02-20T22:03:23.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T22:03:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:03:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:03:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T22:03:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_50 2026-02-20T22:03:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T22:03:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T22:03:23.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:admin_socket: invalid command 2026-02-20T22:03:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:03:23.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_50 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1:3 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ shift 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local val=cluster1:3 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster_var_name=cluster 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local instance_var_name=instance 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=3 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval cluster=cluster1 2026-02-20T22:03:23.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ eval instance=3 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local local_cluster=cluster1:3 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster=cluster1 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local val=cluster1:3 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ local instance_var_name=instance 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ cluster=cluster1 2026-02-20T22:03:23.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ instance=3 2026-02-20T22:03:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T22:03:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval local_cluster=cluster1 2026-02-20T22:03:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ local_cluster=cluster1 2026-02-20T22:03:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ eval instance=3 2026-02-20T22:03:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ instance=3 2026-02-20T22:03:23.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T22:03:24.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:03:24.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:03:24.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T22:03:24.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_50 2026-02-20T22:03:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ return 0 2026-02-20T22:03:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T22:03:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo ' "state": "Replaying"' 2026-02-20T22:03:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ grep -i Replaying 2026-02-20T22:03:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: "state": "Replaying" 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=started 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test started = started 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_50 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = journal ']' 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' snapshot = snapshot ']' 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ mirror_image_snapshot cluster2 mirror image_50 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror/image_50 2026-02-20T22:03:24.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Snapshot ID: 184 2026-02-20T22:03:24.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror mirror image_50 2026-02-20T22:03:24.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_cluster=cluster1 2026-02-20T22:03:24.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:24.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_pool=mirror 2026-02-20T22:03:24.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local remote_pool=mirror 2026-02-20T22:03:24.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:24.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster2-mirror-image_50.status 2026-02-20T22:03:24.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster2-mirror-image_50.status 2026-02-20T22:03:24.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:03:24.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local status_log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_50.status 2026-02-20T22:03:24.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ mkfname cluster1-mirror-image_50.status 2026-02-20T22:03:24.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ echo cluster1-mirror-image_50.status 2026-02-20T22:03:24.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-20T22:03:24.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local local_status_log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.status 2026-02-20T22:03:24.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster2 mirror image_50 /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_50.status 2026-02-20T22:03:24.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:24.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:24.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:24.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_50.status 2026-02-20T22:03:24.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap list --all mirror/image_50 --format xml 2026-02-20T22:03:24.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:24.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-20T22:03:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snapshot_id=184 2026-02-20T22:03:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:03:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.2 2026-02-20T22:03:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.status 2026-02-20T22:03:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.status 2026-02-20T22:03:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-20T22:03:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:25.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=84 2026-02-20T22:03:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 184 = 84 2026-02-20T22:03:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:25.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.4 2026-02-20T22:03:25.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.status 2026-02-20T22:03:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.status 2026-02-20T22:03:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-20T22:03:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:25.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=84 2026-02-20T22:03:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 184 = 84 2026-02-20T22:03:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.8 2026-02-20T22:03:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.status 2026-02-20T22:03:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:26.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.status 2026-02-20T22:03:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-20T22:03:26.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:26.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=84 2026-02-20T22:03:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 184 = 84 2026-02-20T22:03:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T22:03:26.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1.6 2026-02-20T22:03:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ get_newest_mirror_snapshot cluster1 mirror image_50 /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.status 2026-02-20T22:03:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local log=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.status 2026-02-20T22:03:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 snap list --all mirror/image_50 --format xml 2026-02-20T22:03:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-20T22:03:27.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-20T22:03:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local primary_snapshot_id=184 2026-02-20T22:03:27.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 184 = 184 2026-02-20T22:03:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2026-02-20T22:03:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:03:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:03:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T22:03:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:03:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2026-02-20T22:03:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_50': 2026-02-20T22:03:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 128 MiB in 32 objects 2026-02-20T22:03:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:03:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 2 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 10b0221d3e0c 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.10b0221d3e0c 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff, non-primary 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 21:56:11 2026 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 21:56:11 2026 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 21:56:11 2026 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: protected: False 2026-02-20T22:03:29.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:03:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:03:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: f06fe263-7213-4c42-b77b-594ccc26a39b 2026-02-20T22:03:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: false 2026-02-20T22:03:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ compare_image_snaps mirror image_50 snap 2026-02-20T22:03:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:03:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap_name=snap 2026-02-20T22:03:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local ret=0 2026-02-20T22:03:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local rmt_export=/tmp/tmp.2SC0OFePO0/cluster2-mirror-image_50.export 2026-02-20T22:03:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local loc_export=/tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.export 2026-02-20T22:03:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_50.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.export 2026-02-20T22:03:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_50.export 2026-02-20T22:03:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:03:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.export 2026-02-20T22:03:29.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T22:03:29.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cmp /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_50.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.export 2026-02-20T22:03:29.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -f /tmp/tmp.2SC0OFePO0/cluster2-mirror-image_50.export /tmp/tmp.2SC0OFePO0/cluster1-mirror-image_50.export 2026-02-20T22:03:29.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:29.442 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ testlog 'TEST: delete many images' 2026-02-20T22:03:29.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ tee -a /tmp/tmp.2SC0OFePO0/rbd-mirror.test.log 2026-02-20T22:03:29.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ date '+%F %T' 2026-02-20T22:03:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo 2026-02-20 22:03:29 TEST: delete many images 2026-02-20T22:03:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:2026-02-20 22:03:29 TEST: delete many images 2026-02-20T22:03:29.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 50 2026-02-20T22:03:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_1 2026-02-20T22:03:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2026-02-20T22:03:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:03:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2026-02-20T22:03:29.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:29.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-20T22:03:29.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:29.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:29.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:03:29.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:29.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_1 2026-02-20T22:03:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:03:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-20T22:03:29.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-20T22:03:29.909 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:29.901+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61e1a0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:29.909 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:29.901+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee010c80 [3/db87f39c-9375-4487-aa96-a4de9e1499de] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:31.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_2 2026-02-20T22:03:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2026-02-20T22:03:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:03:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:31.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2026-02-20T22:03:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:32.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-20T22:03:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:03:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:32.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:32.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_2 2026-02-20T22:03:32.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:32.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:32.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:03:32.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-20T22:03:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-20T22:03:32.908 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:32.901+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78eb60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:32.908 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:32.901+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe6000 [3/6f61a2e4-1504-47d7-888d-5162115d91ce] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:34.676 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.31834331613990785) > weight (0.25) 2026-02-20T22:03:34.676 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.531311111515354) > weight (0.25) 2026-02-20T22:03:34.676 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-20T22:03:34.677 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-20T22:03:34.677 INFO:tasks.rbd_mirror_thrash:waiting for 8.10665885260221 secs before reviving daemons 2026-02-20T22:03:34.677 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:03:34.669+0000 7fd1d1274640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 32851) UID: 1000 2026-02-20T22:03:34.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_3 2026-02-20T22:03:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2026-02-20T22:03:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:03:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2026-02-20T22:03:35.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-20T22:03:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:03:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_3 2026-02-20T22:03:35.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:35.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:35.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:03:35.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-20T22:03:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-20T22:03:35.920 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:35.913+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78fd40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:35.920 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:35.913+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee010780 [3/ad6926aa-1d75-40e4-80e4-b13fcd076244] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:37.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:37.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_4 2026-02-20T22:03:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2026-02-20T22:03:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:03:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2026-02-20T22:03:38.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-20T22:03:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:03:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_4 2026-02-20T22:03:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:03:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-20T22:03:38.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-20T22:03:38.925 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:38.921+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61e340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:38.925 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:38.921+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7db80 [3/60748c73-296b-4b03-b97e-bb38a2fa0c0a] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_5 2026-02-20T22:03:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2026-02-20T22:03:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:03:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:41.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2026-02-20T22:03:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:41.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-20T22:03:41.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:41.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:41.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:03:41.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:41.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_5 2026-02-20T22:03:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:03:41.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-20T22:03:41.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-20T22:03:41.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:41.928+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61e4e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:41.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:41.928+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7c000 [3/03eab890-30a7-45f9-952e-050bfe363306] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:42.785 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-20T22:03:42.785 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:03:42.785 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-20T22:03:42.785 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-20T22:03:42.786 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-20T22:03:42.788 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-20T22:03:42.788 INFO:tasks.rbd_mirror_thrash:waiting for 80.9067176052247 secs before thrashing 2026-02-20T22:03:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_6 2026-02-20T22:03:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2026-02-20T22:03:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:03:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2026-02-20T22:03:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-20T22:03:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:03:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:44.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_6 2026-02-20T22:03:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:44.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:03:44.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-20T22:03:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-20T22:03:44.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:44.936+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78f380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:44.940 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:44.936+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe6500 [3/73eed487-81fa-4ccf-b6ef-2a93a56ed579] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:46.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_7 2026-02-20T22:03:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2026-02-20T22:03:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:03:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2026-02-20T22:03:47.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-20T22:03:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:03:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:47.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_7 2026-02-20T22:03:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:03:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-20T22:03:47.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-20T22:03:47.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:47.820+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78fba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:47.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:47.820+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee010f00 [3/de4e320f-58cf-4d0e-b463-cea2ccd8e6c3] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_8 2026-02-20T22:03:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2026-02-20T22:03:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:03:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2026-02-20T22:03:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-20T22:03:50.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:50.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:50.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:03:50.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:50.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:50.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_8 2026-02-20T22:03:50.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:50.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:50.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:03:50.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-20T22:03:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-20T22:03:50.947 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:50.944+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61e680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:50.947 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:50.944+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee011900 [3/b550a116-b65a-457c-854a-75ae61a3215e] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:52.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:52.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:52.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_9 2026-02-20T22:03:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2026-02-20T22:03:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:03:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2026-02-20T22:03:53.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-20T22:03:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:53.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:53.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:03:53.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:53.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_9 2026-02-20T22:03:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:03:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-20T22:03:53.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-20T22:03:53.814 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:03:53.812+0000 7fa5e2658640 -1 reset not still connected to 0x55b91ce06a90 2026-02-20T22:03:53.815 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:03:53.812+0000 7fa5e2658640 -1 reset not still connected to 0x55b91eabc680 2026-02-20T22:03:53.816 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:03:53.812+0000 7fa5e2658640 -1 reset not still connected to 0x55b9200475f0 2026-02-20T22:03:53.817 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:03:53.812+0000 7fa5e2658640 -1 reset not still connected to 0x55b920047c70 2026-02-20T22:03:53.817 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:03:53.812+0000 7fa5e2658640 -1 reset not still connected to 0x55b9235c56c0 2026-02-20T22:03:53.817 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:03:53.812+0000 7fa5e2658640 -1 reset not still connected to 0x55b92c1084e0 2026-02-20T22:03:53.818 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:03:53.812+0000 7fa5e2658640 -1 reset not still connected to 0x55b92c109380 2026-02-20T22:03:53.818 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:03:53.812+0000 7fa5e2658640 -1 reset not still connected to 0x55b92c1096c0 2026-02-20T22:03:53.818 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:03:53.812+0000 7fa5e2658640 -1 reset not still connected to 0x55b93467c270 2026-02-20T22:03:53.826 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:53.824+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ed4ab520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:53.826 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:53.824+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7d400 [3/3db04467-75e8-4151-b6e5-29a86d819303] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:54.101 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:54.096+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61f520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:54.102 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:54.096+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7d400 [3/3db04467-75e8-4151-b6e5-29a86d819303] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_10 2026-02-20T22:03:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2026-02-20T22:03:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:03:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2026-02-20T22:03:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:56.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-20T22:03:56.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:56.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:56.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:03:56.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:56.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:56.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_10 2026-02-20T22:03:56.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:56.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:56.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:03:56.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-20T22:03:56.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-20T22:03:56.824 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:56.820+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78fd40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:56.825 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:56.820+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe6780 [3/73f7f3f5-fe4b-491f-8f59-209e2587d369] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:03:58.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:03:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:03:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:03:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_11 2026-02-20T22:03:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2026-02-20T22:03:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T22:03:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:03:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2026-02-20T22:03:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:03:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_11 2026-02-20T22:03:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T22:03:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:03:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:03:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_11 2026-02-20T22:03:59.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:03:59.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:03:59.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T22:03:59.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2026-02-20T22:03:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2026-02-20T22:03:59.828 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:59.824+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ed4aa9c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:03:59.828 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:03:59.824+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee011b80 [3/bb0f4c60-0c8c-41c3-9fc2-067109d74988] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:01.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_12 2026-02-20T22:04:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2026-02-20T22:04:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T22:04:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:01.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2026-02-20T22:04:02.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_12 2026-02-20T22:04:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T22:04:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_12 2026-02-20T22:04:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T22:04:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2026-02-20T22:04:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2026-02-20T22:04:02.833 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:02.828+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ed4aaea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:02.834 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:02.828+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee588000 [3/ffaa636b-9c5c-4539-965b-b5870a492843] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:04.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_13 2026-02-20T22:04:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2026-02-20T22:04:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T22:04:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2026-02-20T22:04:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:05.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_13 2026-02-20T22:04:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T22:04:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:05.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:05.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_13 2026-02-20T22:04:05.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:05.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T22:04:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2026-02-20T22:04:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2026-02-20T22:04:05.855 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:05.852+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc131e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:05.855 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:05.852+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee010500 [3/9cfb07c7-2733-4bb0-9788-9c939376ecdc] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:07.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:07.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:07.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:07.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_14 2026-02-20T22:04:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2026-02-20T22:04:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T22:04:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2026-02-20T22:04:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_14 2026-02-20T22:04:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T22:04:08.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:08.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_14 2026-02-20T22:04:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T22:04:08.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2026-02-20T22:04:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2026-02-20T22:04:08.854 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:08.852+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78e680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:08.854 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:08.852+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7c280 [3/14383d26-7af3-40c8-90a8-27603ddc790c] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_15 2026-02-20T22:04:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2026-02-20T22:04:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T22:04:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2026-02-20T22:04:11.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_15 2026-02-20T22:04:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T22:04:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_15 2026-02-20T22:04:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T22:04:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2026-02-20T22:04:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2026-02-20T22:04:11.861 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:11.856+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78f860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:11.861 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:11.856+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50ea00 [3/d173de17-b1d5-4604-94f9-5797e13957df] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_16 2026-02-20T22:04:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2026-02-20T22:04:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T22:04:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2026-02-20T22:04:14.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_16 2026-02-20T22:04:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T22:04:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_16 2026-02-20T22:04:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T22:04:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2026-02-20T22:04:14.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2026-02-20T22:04:14.866 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:14.864+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc12d00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:14.866 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:14.864+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50f900 [3/f3776a65-963f-4287-81b8-3f98573893e1] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_17 2026-02-20T22:04:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2026-02-20T22:04:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T22:04:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2026-02-20T22:04:17.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_17 2026-02-20T22:04:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T22:04:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_17 2026-02-20T22:04:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T22:04:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2026-02-20T22:04:17.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2026-02-20T22:04:17.830 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:17.828+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc13860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:17.831 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:17.828+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee011400 [3/e65dbfbb-2857-4359-9849-aa0a0d8627ec] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:19.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:19.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:19.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:19.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_18 2026-02-20T22:04:19.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2026-02-20T22:04:19.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:19.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:19.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T22:04:19.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:19.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2026-02-20T22:04:20.613 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5897854507204291) > weight (0.25) 2026-02-20T22:04:20.613 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.6799568904667206) > weight (0.25) 2026-02-20T22:04:20.614 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4540398856164922) > weight (0.25) 2026-02-20T22:04:20.614 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-20T22:04:20.614 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-20T22:04:20.614 INFO:tasks.rbd_mirror_thrash:waiting for 5.740275007183731 secs before reviving daemons 2026-02-20T22:04:20.614 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:04:20.612+0000 7f58fbb5d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 45344) UID: 1000 2026-02-20T22:04:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_18 2026-02-20T22:04:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T22:04:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_18 2026-02-20T22:04:20.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T22:04:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2026-02-20T22:04:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2026-02-20T22:04:20.895 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:20.892+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc13a00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:20.895 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:20.892+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe7900 [3/a7e538f7-880a-4d2e-9bda-105c14b4adf7] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_19 2026-02-20T22:04:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2026-02-20T22:04:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T22:04:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2026-02-20T22:04:23.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_19 2026-02-20T22:04:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T22:04:23.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:23.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:23.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_19 2026-02-20T22:04:23.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:23.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:23.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T22:04:23.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2026-02-20T22:04:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2026-02-20T22:04:23.892 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:23.888+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc13380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:23.892 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:23.888+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe7180 [3/7494457f-5c66-44ae-b871-096697c31154] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:24.105 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:24.100+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edd0f860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:24.105 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:24.100+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe7180 [3/7494457f-5c66-44ae-b871-096697c31154] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:25.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:25.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:25.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:25.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_20 2026-02-20T22:04:25.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2026-02-20T22:04:25.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:25.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:25.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T22:04:25.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:25.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2026-02-20T22:04:26.355 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-20T22:04:26.355 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:04:26.355 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-20T22:04:26.356 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-20T22:04:26.356 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-20T22:04:26.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-20T22:04:26.358 INFO:tasks.rbd_mirror_thrash:waiting for 117.28403515908144 secs before thrashing 2026-02-20T22:04:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_20 2026-02-20T22:04:26.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T22:04:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_20 2026-02-20T22:04:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T22:04:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2026-02-20T22:04:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2026-02-20T22:04:26.905 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:26.899+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc13d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:26.906 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:26.899+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee010000 [3/919065ea-03af-418d-938d-c4f0d0f3196f] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_21 2026-02-20T22:04:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2026-02-20T22:04:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T22:04:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2026-02-20T22:04:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_21 2026-02-20T22:04:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T22:04:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:29.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_21 2026-02-20T22:04:29.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:29.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:29.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T22:04:29.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2026-02-20T22:04:29.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2026-02-20T22:04:29.907 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:29.903+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ed4aab60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:29.907 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:29.903+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50ef00 [3/ea30739c-7e9c-4234-88fe-ea59f7828681] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_22 2026-02-20T22:04:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2026-02-20T22:04:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T22:04:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:31.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2026-02-20T22:04:32.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_22 2026-02-20T22:04:32.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:32.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:32.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T22:04:32.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:32.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_22 2026-02-20T22:04:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T22:04:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2026-02-20T22:04:32.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2026-02-20T22:04:32.824 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:32.819+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ed4ab6c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:32.824 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:32.819+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7d180 [3/34a71a4e-36fa-455c-a493-c55de2c7e753] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:34.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:34.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_23 2026-02-20T22:04:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2026-02-20T22:04:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T22:04:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2026-02-20T22:04:35.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_23 2026-02-20T22:04:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T22:04:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_23 2026-02-20T22:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:35.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T22:04:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2026-02-20T22:04:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2026-02-20T22:04:35.920 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:35.915+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ed4ab1e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:35.920 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:35.915+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7c780 [3/19082fd1-184d-4597-821e-115e896cdf0f] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_24 2026-02-20T22:04:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2026-02-20T22:04:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T22:04:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2026-02-20T22:04:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_24 2026-02-20T22:04:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T22:04:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_24 2026-02-20T22:04:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T22:04:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2026-02-20T22:04:38.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2026-02-20T22:04:38.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:38.927+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc13520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:38.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:38.927+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee010a00 [3/af49b5c9-f8fe-41c3-a35c-c2ad69c2bd7f] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:40.882 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_25 2026-02-20T22:04:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2026-02-20T22:04:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:04:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2026-02-20T22:04:41.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_25 2026-02-20T22:04:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:04:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:41.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_25 2026-02-20T22:04:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:41.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:04:41.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2026-02-20T22:04:41.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2026-02-20T22:04:41.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:41.927+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc12b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:41.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:41.927+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ed779b80 [3/013e07d3-729c-41fe-bfbb-6be668748dac] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:43.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_26 2026-02-20T22:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2026-02-20T22:04:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:04:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2026-02-20T22:04:44.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_26 2026-02-20T22:04:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:04:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_26 2026-02-20T22:04:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:04:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2026-02-20T22:04:44.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2026-02-20T22:04:44.836 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:44.831+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edd0f860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:44.836 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:44.831+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50f180 [3/eebea35a-dfd0-4c97-aade-b52b3cada4e8] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:46.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_27 2026-02-20T22:04:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2026-02-20T22:04:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:04:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2026-02-20T22:04:47.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_27 2026-02-20T22:04:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:04:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:47.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_27 2026-02-20T22:04:47.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:47.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:47.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:04:47.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2026-02-20T22:04:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2026-02-20T22:04:47.837 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:47.831+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edd0e680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:47.837 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:47.831+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7ca00 [3/1b8e35e9-ec45-44f4-a0d7-dbb42eabaaa7] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_28 2026-02-20T22:04:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2026-02-20T22:04:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:04:49.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2026-02-20T22:04:50.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_28 2026-02-20T22:04:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:04:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_28 2026-02-20T22:04:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:04:50.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2026-02-20T22:04:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2026-02-20T22:04:50.842 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:50.839+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ed4aa9c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:50.842 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:50.839+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe7b80 [3/ab5b8a6b-3393-4121-92ba-22af9b8d3ce4] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_29 2026-02-20T22:04:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2026-02-20T22:04:52.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:04:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:52.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2026-02-20T22:04:53.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_29 2026-02-20T22:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:53.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_29 2026-02-20T22:04:53.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:53.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:53.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:04:53.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2026-02-20T22:04:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2026-02-20T22:04:53.852 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:53.847+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78ed00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:53.852 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:53.847+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7cf00 [3/2d03f9ea-eefd-44d9-b8bd-e230886e2bc8] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:54.098 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:54.091+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ed4aad00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:54.098 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:54.091+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7cf00 [3/2d03f9ea-eefd-44d9-b8bd-e230886e2bc8] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:55.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_30 2026-02-20T22:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2026-02-20T22:04:55.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:55.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2026-02-20T22:04:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_30 2026-02-20T22:04:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:04:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:56.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:56.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_30 2026-02-20T22:04:56.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:56.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:56.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:04:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2026-02-20T22:04:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2026-02-20T22:04:57.066 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:57.063+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78eea0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:57.066 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:57.063+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50ec80 [3/d649a580-cc10-448a-bdd7-2ee6987b9a69] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:04:58.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:04:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:04:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:04:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_31 2026-02-20T22:04:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2026-02-20T22:04:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:04:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:04:58.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2026-02-20T22:04:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:04:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_31 2026-02-20T22:04:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:04:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:04:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:04:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_31 2026-02-20T22:04:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:04:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:04:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:04:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2026-02-20T22:04:59.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2026-02-20T22:04:59.862 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:59.859+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61e680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:04:59.862 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:04:59.859+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7cc80 [3/28720fa9-2640-4d05-b3cb-cf921814d23d] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:01.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_32 2026-02-20T22:05:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2026-02-20T22:05:01.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:01.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:01.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:05:01.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:01.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2026-02-20T22:05:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_32 2026-02-20T22:05:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:05:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:02.728 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_32 2026-02-20T22:05:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:05:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2026-02-20T22:05:02.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2026-02-20T22:05:02.838 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:02.831+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78eb60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:02.838 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:02.831+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe6280 [3/6f7f0c8e-e585-45a6-922c-3d3abe3d38f8] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:03.695 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.38064465541316306) > weight (0.25) 2026-02-20T22:05:03.695 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-20T22:05:03.696 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-20T22:05:03.696 INFO:tasks.rbd_mirror_thrash:waiting for 7.088275214590603 secs before reviving daemons 2026-02-20T22:05:03.696 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:05:03.691+0000 7efcc3aa1640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 29122) UID: 1000 2026-02-20T22:05:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_33 2026-02-20T22:05:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2026-02-20T22:05:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:05:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2026-02-20T22:05:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:05.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_33 2026-02-20T22:05:05.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:05.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:05.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:05:05.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:05.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_33 2026-02-20T22:05:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:05:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2026-02-20T22:05:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2026-02-20T22:05:05.881 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:05.875+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78e4e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:05.881 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:05.875+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe7680 [3/9a59b858-296a-4d6b-9785-36f62034ac7a] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_34 2026-02-20T22:05:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2026-02-20T22:05:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:05:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:07.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2026-02-20T22:05:08.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:08.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_34 2026-02-20T22:05:08.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:08.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:08.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:05:08.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:08.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:08.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_34 2026-02-20T22:05:08.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:08.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:08.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:05:08.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2026-02-20T22:05:08.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2026-02-20T22:05:08.890 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:08.883+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78e820 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:08.890 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:08.883+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe6a00 [3/843abf52-674b-43d7-b959-4287ddcbd20a] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:10.785 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-20T22:05:10.785 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:05:10.785 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-20T22:05:10.785 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T22:05:10.786 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T22:05:10.788 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T22:05:10.788 INFO:tasks.rbd_mirror_thrash:waiting for 104.99016702151455 secs before thrashing 2026-02-20T22:05:10.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_35 2026-02-20T22:05:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2026-02-20T22:05:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:05:10.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:10.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2026-02-20T22:05:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_35 2026-02-20T22:05:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:05:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:11.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:11.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_35 2026-02-20T22:05:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:05:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2026-02-20T22:05:11.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2026-02-20T22:05:11.891 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:11.886+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78fd40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:11.891 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:11.886+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7c500 [3/15ffbe72-993b-43ba-9ff0-25518ecb6ae4] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_36 2026-02-20T22:05:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2026-02-20T22:05:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:05:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2026-02-20T22:05:14.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_36 2026-02-20T22:05:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:05:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:14.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_36 2026-02-20T22:05:14.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:14.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:14.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:05:14.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2026-02-20T22:05:14.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2026-02-20T22:05:14.911 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:14.906+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78f380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:14.911 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:14.906+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50e280 [3/cb656abb-b533-4a24-8070-aceb29417b9d] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:16.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:16.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:16.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_37 2026-02-20T22:05:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2026-02-20T22:05:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:05:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:16.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2026-02-20T22:05:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_37 2026-02-20T22:05:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:05:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_37 2026-02-20T22:05:17.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:17.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:17.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:05:17.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2026-02-20T22:05:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2026-02-20T22:05:17.836 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:17.830+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78fba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:17.836 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:17.830+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7d680 [3/433c55c5-23ec-4be7-a8a5-59991ae7abd8] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:19.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_38 2026-02-20T22:05:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2026-02-20T22:05:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:05:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2026-02-20T22:05:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:20.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_38 2026-02-20T22:05:20.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:20.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:20.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:05:20.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:20.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_38 2026-02-20T22:05:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:05:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2026-02-20T22:05:20.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2026-02-20T22:05:20.915 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:20.910+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc13ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:20.915 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:20.910+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50f680 [3/f0d4e737-438c-4247-8539-7b6130b2be4d] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:22.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_39 2026-02-20T22:05:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2026-02-20T22:05:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:05:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:22.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2026-02-20T22:05:23.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_39 2026-02-20T22:05:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:05:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:23.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_39 2026-02-20T22:05:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:05:23.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2026-02-20T22:05:23.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2026-02-20T22:05:23.924 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:23.918+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78e680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:23.924 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:23.918+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe7400 [3/876e1da9-8a7e-447b-81f0-6c8df3b23c0c] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:24.097 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:24.090+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edc13860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:24.097 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:24.090+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe7400 [3/876e1da9-8a7e-447b-81f0-6c8df3b23c0c] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_40 2026-02-20T22:05:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2026-02-20T22:05:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:05:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2026-02-20T22:05:26.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_40 2026-02-20T22:05:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:05:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_40 2026-02-20T22:05:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:05:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2026-02-20T22:05:26.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2026-02-20T22:05:26.932 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:26.926+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78f860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:26.933 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:26.926+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50e000 [3/c4c8b7cd-d483-42c9-9489-fa930aaeae6b] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:28.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_41 2026-02-20T22:05:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2026-02-20T22:05:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:05:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2026-02-20T22:05:29.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_41 2026-02-20T22:05:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:05:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_41 2026-02-20T22:05:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:05:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2026-02-20T22:05:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2026-02-20T22:05:29.941 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:29.934+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0eddb7860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:29.941 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:29.934+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50fb80 [3/f3be01dc-d6c3-44a9-9f77-d87df8a565ca] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:31.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_42 2026-02-20T22:05:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2026-02-20T22:05:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:05:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:31.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2026-02-20T22:05:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_42 2026-02-20T22:05:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:05:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:32.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_42 2026-02-20T22:05:32.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:05:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2026-02-20T22:05:32.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2026-02-20T22:05:32.838 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:32.834+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0eddb7ba0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:32.838 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:32.834+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee011180 [3/e3fc513d-202f-403b-8d2f-5ae273783233] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:34.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_43 2026-02-20T22:05:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2026-02-20T22:05:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:05:34.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:34.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2026-02-20T22:05:35.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_43 2026-02-20T22:05:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:05:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_43 2026-02-20T22:05:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:05:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2026-02-20T22:05:35.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2026-02-20T22:05:35.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:35.934+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0eddb7a00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:35.939 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:35.934+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee011680 [3/b3411393-1507-494e-b3af-7d1d3d485652] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_44 2026-02-20T22:05:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2026-02-20T22:05:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:05:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2026-02-20T22:05:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_44 2026-02-20T22:05:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:05:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_44 2026-02-20T22:05:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:05:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2026-02-20T22:05:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2026-02-20T22:05:38.951 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:38.946+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78f520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:38.952 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:38.946+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee010280 [3/9c7289c8-3641-48c4-aeec-d1a3caf6cc21] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:40.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:40.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_45 2026-02-20T22:05:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2026-02-20T22:05:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:05:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2026-02-20T22:05:41.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_45 2026-02-20T22:05:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:05:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:41.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_45 2026-02-20T22:05:41.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:41.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:41.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:05:41.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2026-02-20T22:05:41.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2026-02-20T22:05:41.962 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:41.954+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78f6c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:41.962 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:41.954+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50e500 [3/cf134cc1-67a2-4edd-85cf-ecb315b92e21] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:43.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:43.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:43.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:43.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_46 2026-02-20T22:05:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2026-02-20T22:05:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:05:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:43.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2026-02-20T22:05:44.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_46 2026-02-20T22:05:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:05:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_46 2026-02-20T22:05:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:05:44.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2026-02-20T22:05:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2026-02-20T22:05:45.042 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:45.038+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0eddb7d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:45.042 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:45.038+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe6c80 [3/8e7ce785-f9ba-4268-a043-f8b326632069] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_47 2026-02-20T22:05:46.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2026-02-20T22:05:46.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:46.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:46.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:05:46.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:46.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2026-02-20T22:05:47.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:47.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_47 2026-02-20T22:05:47.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:47.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:47.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:05:47.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:47.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_47 2026-02-20T22:05:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:05:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2026-02-20T22:05:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2026-02-20T22:05:47.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:47.842+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0eddb71e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:47.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:47.842+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe6f00 [3/740d48a2-b3be-43ea-a507-e1c8ff34cb29] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:47.884 INFO:tasks.ceph.cluster1.osd.2.trial032.stderr:2026-02-20T22:05:47.818+0000 7f80540a9640 -1 reset not still connected to 0x5620a2f33930 2026-02-20T22:05:47.884 INFO:tasks.ceph.cluster1.osd.2.trial032.stderr:2026-02-20T22:05:47.818+0000 7f80540a9640 -1 reset not still connected to 0x5620a3e2a1a0 2026-02-20T22:05:47.884 INFO:tasks.ceph.cluster1.osd.2.trial032.stderr:2026-02-20T22:05:47.818+0000 7f80540a9640 -1 reset not still connected to 0x5620a538f860 2026-02-20T22:05:49.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:49.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:49.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:49.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_48 2026-02-20T22:05:49.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2026-02-20T22:05:49.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:49.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:49.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:05:49.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:49.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2026-02-20T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:50.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_48 2026-02-20T22:05:50.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:50.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:05:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:50.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_48 2026-02-20T22:05:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:05:50.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2026-02-20T22:05:50.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2026-02-20T22:05:50.867 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:50.862+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edd0f1e0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:50.867 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:50.862+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7d900 [3/4a44519a-03e3-4a42-bf80-6ed4a8037a65] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:52.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:52.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:52.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_49 2026-02-20T22:05:52.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2026-02-20T22:05:52.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:52.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:52.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:05:52.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:52.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2026-02-20T22:05:53.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:53.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_49 2026-02-20T22:05:53.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:53.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:05:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:53.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_49 2026-02-20T22:05:53.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:53.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:53.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:05:53.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2026-02-20T22:05:53.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2026-02-20T22:05:53.872 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:53.866+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78f040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:53.873 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:53.866+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50e780 [3/d0428f4e-f589-457e-8185-f6b0bba89f12] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:54.095 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:54.090+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edd0f380 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:54.095 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:54.090+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50e780 [3/d0428f4e-f589-457e-8185-f6b0bba89f12] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:55.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_50 2026-02-20T22:05:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2026-02-20T22:05:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:05:55.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local snap=snap 2026-02-20T22:05:55.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2026-02-20T22:05:56.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing snap: 100% complete...done. 2026-02-20T22:05:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_50 2026-02-20T22:05:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:05:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:05:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:05:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_50 2026-02-20T22:05:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:05:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:05:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2026-02-20T22:05:56.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2026-02-20T22:05:56.865 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:56.858+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee78e9c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:05:56.865 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:05:56.858+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50f400 [3/f06fe263-7213-4c42-b77b-594ccc26a39b] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:05:58.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T22:05:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ testlog 'TEST: image deletions should propagate' 2026-02-20T22:05:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ tee -a /tmp/tmp.2SC0OFePO0/rbd-mirror.test.log 2026-02-20T22:05:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ date '+%F %T' 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo 2026-02-20 22:05:58 TEST: image deletions should propagate 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:2026-02-20 22:05:58 TEST: image deletions should propagate 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_pool_images cluster1 mirror 0 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_count=0 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local count 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local last_count=0 2026-02-20T22:05:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ true 2026-02-20T22:05:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 40 2026-02-20T22:05:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in `seq 1 40` 2026-02-20T22:05:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 1 -ne 1 2026-02-20T22:05:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T22:05:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep 'images: ' 2026-02-20T22:05:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cut '-d ' -f 2 2026-02-20T22:05:58.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T22:05:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ count=0 2026-02-20T22:05:58.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 = 0 2026-02-20T22:05:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2026-02-20T22:05:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:05:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state 2026-02-20T22:05:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 40 2026-02-20T22:05:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in `seq 1 40` 2026-02-20T22:05:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 1 -ne 1 2026-02-20T22:05:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T22:05:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep 'image health:' 2026-02-20T22:05:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cut '-d ' -f 3 2026-02-20T22:05:58.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T22:05:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ state=OK 2026-02-20T22:05:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test OK = ERROR 2026-02-20T22:05:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test OK = OK 2026-02-20T22:05:58.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 50 2026-02-20T22:05:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_1 2026-02-20T22:05:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-20T22:05:58.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_1 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T22:05:58.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:58.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-20T22:05:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:05:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:05:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:05:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-20T22:05:58.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_1 2026-02-20T22:05:58.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T22:05:58.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-20T22:05:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:05:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_2 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:05:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T22:05:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_2 2026-02-20T22:05:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T22:05:59.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T22:05:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:05:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:05:59.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:05:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-20T22:05:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:05:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:05:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:05:59.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:05:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T22:05:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_2 2026-02-20T22:05:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T22:05:59.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_3 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_3 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T22:05:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-20T22:05:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:05:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:05:59.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:05:59.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-20T22:05:59.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_3 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T22:05:59.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-20T22:05:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:05:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:05:59.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_4 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T22:05:59.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_4 2026-02-20T22:05:59.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T22:05:59.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-20T22:05:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:05:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:05:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:05:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-20T22:05:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_4 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T22:05:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-20T22:05:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:05:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:05:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_5 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T22:05:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_5 2026-02-20T22:05:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T22:05:59.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-20T22:05:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:05:59.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:05:59.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:05:59.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_5 2026-02-20T22:05:59.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T22:05:59.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-20T22:05:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:05:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:05:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_6 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T22:05:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_6 2026-02-20T22:05:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T22:05:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-20T22:05:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:05:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:05:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:05:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-20T22:05:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:05:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:05:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:05:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:05:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T22:05:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_6 2026-02-20T22:05:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T22:05:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-20T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_7 2026-02-20T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-20T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_7 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:05:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-20T22:05:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:05:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:05:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-20T22:06:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:06:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:00.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T22:06:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_7 2026-02-20T22:06:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T22:06:00.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-20T22:06:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_8 2026-02-20T22:06:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-20T22:06:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_8 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T22:06:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-20T22:06:00.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:00.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:00.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-20T22:06:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:06:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T22:06:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_8 2026-02-20T22:06:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T22:06:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-20T22:06:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:00.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_9 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_9 2026-02-20T22:06:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T22:06:00.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-20T22:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:00.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-20T22:06:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:06:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T22:06:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_9 2026-02-20T22:06:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T22:06:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-20T22:06:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_10 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T22:06:00.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_10 2026-02-20T22:06:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T22:06:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-20T22:06:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:00.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-20T22:06:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:06:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T22:06:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_10 2026-02-20T22:06:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T22:06:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-20T22:06:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_11 2026-02-20T22:06:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_11 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_11 2026-02-20T22:06:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-20T22:06:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-20T22:06:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:00.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_11 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_11 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_11 2026-02-20T22:06:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-20T22:06:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-20T22:06:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_12 2026-02-20T22:06:00.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_12 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_12 2026-02-20T22:06:00.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-20T22:06:00.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-20T22:06:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2026-02-20T22:06:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_12 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_12 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_12 2026-02-20T22:06:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-20T22:06:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-20T22:06:00.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:00.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:00.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:00.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_13 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_13 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_13 2026-02-20T22:06:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-20T22:06:00.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-20T22:06:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:00.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2026-02-20T22:06:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_13 2026-02-20T22:06:00.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_13 2026-02-20T22:06:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_13 2026-02-20T22:06:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-20T22:06:00.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-20T22:06:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_14 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T22:06:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_14 2026-02-20T22:06:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_14 2026-02-20T22:06:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-20T22:06:01.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-20T22:06:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:01.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2026-02-20T22:06:01.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_14 2026-02-20T22:06:01.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:01.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:01.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:01.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_14 2026-02-20T22:06:01.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_14 2026-02-20T22:06:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-20T22:06:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-20T22:06:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:01.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_15 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_15 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_15 2026-02-20T22:06:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-20T22:06:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-20T22:06:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2026-02-20T22:06:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_15 2026-02-20T22:06:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:01.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_15 2026-02-20T22:06:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_15 2026-02-20T22:06:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-20T22:06:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_16 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T22:06:01.327 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_16 2026-02-20T22:06:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_16 2026-02-20T22:06:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-20T22:06:01.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-20T22:06:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:01.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2026-02-20T22:06:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_16 2026-02-20T22:06:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:01.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_16 2026-02-20T22:06:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_16 2026-02-20T22:06:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-20T22:06:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-20T22:06:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_17 2026-02-20T22:06:01.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_17 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_17 2026-02-20T22:06:01.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-20T22:06:01.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-20T22:06:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:01.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2026-02-20T22:06:01.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_17 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_17 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_17 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-20T22:06:01.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-20T22:06:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_18 2026-02-20T22:06:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2026-02-20T22:06:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_18 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_18 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-20T22:06:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-20T22:06:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_18 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_18 2026-02-20T22:06:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_18 2026-02-20T22:06:01.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-20T22:06:01.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-20T22:06:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:01.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_19 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:01.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_19 2026-02-20T22:06:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_19 2026-02-20T22:06:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-20T22:06:01.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-20T22:06:01.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:01.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:01.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2026-02-20T22:06:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_19 2026-02-20T22:06:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_19 2026-02-20T22:06:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_19 2026-02-20T22:06:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-20T22:06:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-20T22:06:01.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:01.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_20 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:01.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_20 2026-02-20T22:06:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_20 2026-02-20T22:06:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-20T22:06:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:01.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-20T22:06:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2026-02-20T22:06:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_20 2026-02-20T22:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_20 2026-02-20T22:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_20 2026-02-20T22:06:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-20T22:06:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-20T22:06:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_21 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_21 2026-02-20T22:06:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_21 2026-02-20T22:06:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-20T22:06:02.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-20T22:06:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:02.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2026-02-20T22:06:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_21 2026-02-20T22:06:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_21 2026-02-20T22:06:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_21 2026-02-20T22:06:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-20T22:06:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-20T22:06:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_22 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_22 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_22 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-20T22:06:02.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-20T22:06:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2026-02-20T22:06:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_22 2026-02-20T22:06:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_22 2026-02-20T22:06:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_22 2026-02-20T22:06:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-20T22:06:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-20T22:06:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:02.389 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_23 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:02.390 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_23 2026-02-20T22:06:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_23 2026-02-20T22:06:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-20T22:06:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-20T22:06:02.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:02.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:02.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2026-02-20T22:06:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_23 2026-02-20T22:06:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_23 2026-02-20T22:06:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_23 2026-02-20T22:06:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-20T22:06:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-20T22:06:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_24 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:02.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_24 2026-02-20T22:06:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_24 2026-02-20T22:06:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-20T22:06:02.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-20T22:06:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:02.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_24 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_24 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_24 2026-02-20T22:06:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-20T22:06:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-20T22:06:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_25 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_25 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_25 2026-02-20T22:06:02.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-20T22:06:02.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-20T22:06:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2026-02-20T22:06:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_25 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_25 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_25 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-20T22:06:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-20T22:06:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_26 2026-02-20T22:06:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2026-02-20T22:06:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_26 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_26 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-20T22:06:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-20T22:06:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:02.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:02.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2026-02-20T22:06:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_26 2026-02-20T22:06:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_26 2026-02-20T22:06:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_26 2026-02-20T22:06:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-20T22:06:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:02.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-20T22:06:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_27 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_27 2026-02-20T22:06:02.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:02.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:02.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_27 2026-02-20T22:06:02.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-20T22:06:02.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-20T22:06:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:03.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:03.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2026-02-20T22:06:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_27 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_27 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_27 2026-02-20T22:06:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-20T22:06:03.122 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-20T22:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_28 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_28 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_28 2026-02-20T22:06:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-20T22:06:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-20T22:06:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2026-02-20T22:06:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_28 2026-02-20T22:06:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_28 2026-02-20T22:06:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_28 2026-02-20T22:06:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-20T22:06:03.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-20T22:06:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_29 2026-02-20T22:06:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2026-02-20T22:06:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_29 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_29 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-20T22:06:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-20T22:06:03.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2026-02-20T22:06:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_29 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_29 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_29 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-20T22:06:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-20T22:06:03.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:03.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:03.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:03.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:03.444 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_30 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_30 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_30 2026-02-20T22:06:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-20T22:06:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-20T22:06:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:03.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2026-02-20T22:06:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_30 2026-02-20T22:06:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_30 2026-02-20T22:06:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_30 2026-02-20T22:06:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-20T22:06:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.590 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-20T22:06:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_31 2026-02-20T22:06:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2026-02-20T22:06:03.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_31 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_31 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-20T22:06:03.595 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-20T22:06:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2026-02-20T22:06:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_31 2026-02-20T22:06:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_31 2026-02-20T22:06:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_31 2026-02-20T22:06:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-20T22:06:03.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-20T22:06:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_32 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_32 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.747 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_32 2026-02-20T22:06:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-20T22:06:03.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-20T22:06:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2026-02-20T22:06:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_32 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_32 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_32 2026-02-20T22:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-20T22:06:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:03.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_33 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:06:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_33 2026-02-20T22:06:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_33 2026-02-20T22:06:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-20T22:06:03.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-20T22:06:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2026-02-20T22:06:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_33 2026-02-20T22:06:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:04.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_33 2026-02-20T22:06:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_33 2026-02-20T22:06:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-20T22:06:04.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-20T22:06:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:04.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_34 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_34 2026-02-20T22:06:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_34 2026-02-20T22:06:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-20T22:06:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-20T22:06:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:04.280 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2026-02-20T22:06:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_34 2026-02-20T22:06:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_34 2026-02-20T22:06:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_34 2026-02-20T22:06:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-20T22:06:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-20T22:06:04.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:04.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_35 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_35 2026-02-20T22:06:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_35 2026-02-20T22:06:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-20T22:06:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-20T22:06:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2026-02-20T22:06:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_35 2026-02-20T22:06:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_35 2026-02-20T22:06:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_35 2026-02-20T22:06:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-20T22:06:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.554 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-20T22:06:04.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_36 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:04.559 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:04.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_36 2026-02-20T22:06:04.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_36 2026-02-20T22:06:04.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-20T22:06:04.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-20T22:06:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:04.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2026-02-20T22:06:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_36 2026-02-20T22:06:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:04.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_36 2026-02-20T22:06:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_36 2026-02-20T22:06:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-20T22:06:04.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-20T22:06:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_37 2026-02-20T22:06:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2026-02-20T22:06:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_37 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_37 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-20T22:06:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-20T22:06:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:04.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2026-02-20T22:06:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_37 2026-02-20T22:06:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_37 2026-02-20T22:06:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_37 2026-02-20T22:06:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-20T22:06:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-20T22:06:04.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:04.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_38 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:04.862 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_38 2026-02-20T22:06:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_38 2026-02-20T22:06:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-20T22:06:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:04.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-20T22:06:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:04.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2026-02-20T22:06:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:04.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_38 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_38 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_38 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-20T22:06:04.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:05.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-20T22:06:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_39 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_39 2026-02-20T22:06:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_39 2026-02-20T22:06:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-20T22:06:05.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-20T22:06:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2026-02-20T22:06:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_39 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_39 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_39 2026-02-20T22:06:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-20T22:06:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:05.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-20T22:06:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_40 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:05.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_40 2026-02-20T22:06:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_40 2026-02-20T22:06:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-20T22:06:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:05.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-20T22:06:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2026-02-20T22:06:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:05.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_40 2026-02-20T22:06:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_40 2026-02-20T22:06:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:05.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_40 2026-02-20T22:06:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-20T22:06:05.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_41 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_41 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_41 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-20T22:06:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:05.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-20T22:06:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2026-02-20T22:06:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_41 2026-02-20T22:06:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:05.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_41 2026-02-20T22:06:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_41 2026-02-20T22:06:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-20T22:06:05.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-20T22:06:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_42 2026-02-20T22:06:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2026-02-20T22:06:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_42 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_42 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-20T22:06:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-20T22:06:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2026-02-20T22:06:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_42 2026-02-20T22:06:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_42 2026-02-20T22:06:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_42 2026-02-20T22:06:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-20T22:06:06.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-20T22:06:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_43 2026-02-20T22:06:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2026-02-20T22:06:06.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_43 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_43 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-20T22:06:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-20T22:06:06.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:06.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:06.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2026-02-20T22:06:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_43 2026-02-20T22:06:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:06.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_43 2026-02-20T22:06:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_43 2026-02-20T22:06:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-20T22:06:06.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:06.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-20T22:06:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:06.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_44 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_44 2026-02-20T22:06:06.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_44 2026-02-20T22:06:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-20T22:06:06.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:06.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-20T22:06:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_44 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_44 2026-02-20T22:06:06.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_44 2026-02-20T22:06:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-20T22:06:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-20T22:06:06.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_45 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:06.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:06.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:06.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:06.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_45 2026-02-20T22:06:06.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:06.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:06.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_45 2026-02-20T22:06:06.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-20T22:06:06.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-20T22:06:06.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:06.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:06.561 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_45 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_45 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_45 2026-02-20T22:06:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-20T22:06:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:06.685 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-20T22:06:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_46 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_46 2026-02-20T22:06:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_46 2026-02-20T22:06:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-20T22:06:06.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-20T22:06:06.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2026-02-20T22:06:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_46 2026-02-20T22:06:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_46 2026-02-20T22:06:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_46 2026-02-20T22:06:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-20T22:06:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_47 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_47 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_47 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-20T22:06:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-20T22:06:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:07.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2026-02-20T22:06:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_47 2026-02-20T22:06:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_47 2026-02-20T22:06:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_47 2026-02-20T22:06:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-20T22:06:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:07.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-20T22:06:07.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_48 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:06:07.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_48 2026-02-20T22:06:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_48 2026-02-20T22:06:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-20T22:06:07.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:07.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-20T22:06:07.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:07.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:07.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:07.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2026-02-20T22:06:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_48 2026-02-20T22:06:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:07.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_48 2026-02-20T22:06:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_48 2026-02-20T22:06:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-20T22:06:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:07.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-20T22:06:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_49 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:06:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_49 2026-02-20T22:06:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_49 2026-02-20T22:06:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-20T22:06:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:07.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-20T22:06:07.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:07.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:07.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2026-02-20T22:06:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_49 2026-02-20T22:06:07.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_49 2026-02-20T22:06:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_49 2026-02-20T22:06:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-20T22:06:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:07.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-20T22:06:07.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:07.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:07.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:07.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:07.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T22:06:07.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_50 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_50 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:07.700 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_50 2026-02-20T22:06:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-20T22:06:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-20T22:06:07.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:07.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:07.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_50 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_50 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_50 2026-02-20T22:06:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-20T22:06:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:07.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-20T22:06:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:07.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ testlog 'TEST: delete images during bootstrap' 2026-02-20T22:06:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ tee -a /tmp/tmp.2SC0OFePO0/rbd-mirror.test.log 2026-02-20T22:06:07.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ date '+%F %T' 2026-02-20T22:06:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo 2026-02-20 22:06:07 TEST: delete images during bootstrap 2026-02-20T22:06:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:2026-02-20 22:06:07 TEST: delete images during bootstrap 2026-02-20T22:06:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_pool_mirror_mode cluster1 mirror image 2026-02-20T22:06:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=image 2026-02-20T22:06:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2026-02-20T22:06:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd: mirroring is already configured for image mode 2026-02-20T22:06:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-20T22:06:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=image 2026-02-20T22:06:07.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-20T22:06:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd: mirroring is already configured for image mode 2026-02-20T22:06:07.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ start_mirror cluster1 2026-02-20T22:06:07.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set_cluster_instance cluster1 cluster instance 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local val=cluster1 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster_var_name=cluster 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local instance_var_name=instance 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cluster=cluster1 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=cluster1 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' cluster1 = cluster1 ']' 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ instance=0 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval cluster=cluster1 2026-02-20T22:06:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ cluster=cluster1 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ eval instance=0 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ instance=0 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=test 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 1 10 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_1 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_1 512M 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:06:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:06:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:06:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:06:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:06:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:06:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_1 2026-02-20T22:06:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_1 2026-02-20T22:06:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:06:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 snapshot 2026-02-20T22:06:08.750 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:06:08.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-02-20T22:06:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_1': 2026-02-20T22:06:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:06:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:06:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:06:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 18e9492fc4ef 2026-02-20T22:06:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.18e9492fc4ef 2026-02-20T22:06:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:06:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:06:08.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:06:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:06:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:06:07 2026 2026-02-20T22:06:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:06:07 2026 2026-02-20T22:06:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:06:07 2026 2026-02-20T22:06:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:06:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:06:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 2934a2ff-785b-412e-b7a1-904a2fa7c062 2026-02-20T22:06:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:06:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-20T22:06:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:06:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=8 2026-02-20T22:06:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:06:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:06:08.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 39149568 bytes 2026-02-20T22:06:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 168104960 bytes 2026-02-20T22:06:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 224387072 bytes 2026-02-20T22:06:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 262896640 bytes 2026-02-20T22:06:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 306891776 bytes 2026-02-20T22:06:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 355643392 bytes 2026-02-20T22:06:09.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 395131904 bytes 2026-02-20T22:06:09.311 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 436968448 bytes 2026-02-20T22:06:09.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 463299584 bytes 2026-02-20T22:06:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 528218112 bytes 2026-02-20T22:06:09.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 560225280 bytes 2026-02-20T22:06:09.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 608929792 bytes 2026-02-20T22:06:09.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 636257280 bytes 2026-02-20T22:06:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 716179456 bytes 2026-02-20T22:06:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 765146112 bytes 2026-02-20T22:06:09.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 907835392 bytes 2026-02-20T22:06:09.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 991959040 bytes 2026-02-20T22:06:09.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 1071824896 bytes 2026-02-20T22:06:09.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 1135380480 bytes 2026-02-20T22:06:09.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1156873216 bytes 2026-02-20T22:06:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1256698880 bytes 2026-02-20T22:06:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1323971584 bytes 2026-02-20T22:06:10.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1365457920 bytes 2026-02-20T22:06:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1393319936 bytes 2026-02-20T22:06:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1419441152 bytes 2026-02-20T22:06:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1489201152 bytes 2026-02-20T22:06:10.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1567435776 bytes 2026-02-20T22:06:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1601201152 bytes 2026-02-20T22:06:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1660600320 bytes 2026-02-20T22:06:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1691575296 bytes 2026-02-20T22:06:10.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1748483072 bytes 2026-02-20T22:06:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1756896256 bytes 2026-02-20T22:06:10.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1835209728 bytes 2026-02-20T22:06:10.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1892877312 bytes 2026-02-20T22:06:10.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1959523328 bytes 2026-02-20T22:06:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 2009447424 bytes 2026-02-20T22:06:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 2100438016 bytes 2026-02-20T22:06:10.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 2122989568 bytes 2026-02-20T22:06:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 2158569472 bytes 2026-02-20T22:06:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 2162772992 bytes 2026-02-20T22:06:11.042 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 2218595328 bytes 2026-02-20T22:06:11.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2219204608 bytes 2026-02-20T22:06:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2284523520 bytes 2026-02-20T22:06:11.186 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2360778752 bytes 2026-02-20T22:06:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2361389056 bytes 2026-02-20T22:06:11.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2428140544 bytes 2026-02-20T22:06:11.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2495087616 bytes 2026-02-20T22:06:11.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2553949184 bytes 2026-02-20T22:06:11.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2565734400 bytes 2026-02-20T22:06:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2661536768 bytes 2026-02-20T22:06:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2677645312 bytes 2026-02-20T22:06:11.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2687219712 bytes 2026-02-20T22:06:11.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2710841344 bytes 2026-02-20T22:06:11.641 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2717802496 bytes 2026-02-20T22:06:11.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2727903232 bytes 2026-02-20T22:06:11.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2796858368 bytes 2026-02-20T22:06:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2871936000 bytes 2026-02-20T22:06:11.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2928009216 bytes 2026-02-20T22:06:11.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2928664576 bytes 2026-02-20T22:06:11.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 3021370368 bytes 2026-02-20T22:06:12.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 3078170624 bytes 2026-02-20T22:06:12.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 3092907008 bytes 2026-02-20T22:06:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 3112915968 bytes 2026-02-20T22:06:12.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3171466240 bytes 2026-02-20T22:06:12.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3227432960 bytes 2026-02-20T22:06:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3272671232 bytes 2026-02-20T22:06:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3345862656 bytes 2026-02-20T22:06:12.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3389770752 bytes 2026-02-20T22:06:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3433005056 bytes 2026-02-20T22:06:12.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3477287936 bytes 2026-02-20T22:06:12.503 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3552683008 bytes 2026-02-20T22:06:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3582998528 bytes 2026-02-20T22:06:12.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3677600768 bytes 2026-02-20T22:06:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3697334272 bytes 2026-02-20T22:06:12.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3736399872 bytes 2026-02-20T22:06:12.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3800735744 bytes 2026-02-20T22:06:12.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3823802368 bytes 2026-02-20T22:06:12.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3854833664 bytes 2026-02-20T22:06:12.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3914027008 bytes 2026-02-20T22:06:12.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3987301376 bytes 2026-02-20T22:06:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 4025153536 bytes 2026-02-20T22:06:13.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 4029813760 bytes 2026-02-20T22:06:13.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 4105140224 bytes 2026-02-20T22:06:13.127 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 4114462720 bytes 2026-02-20T22:06:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 4210243584 bytes 2026-02-20T22:06:13.237 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 4240766976 bytes 2026-02-20T22:06:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 4250699776 bytes 2026-02-20T22:06:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4278253568 bytes 2026-02-20T22:06:13.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4299675648 bytes 2026-02-20T22:06:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4338646016 bytes 2026-02-20T22:06:13.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4367984640 bytes 2026-02-20T22:06:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4477589504 bytes 2026-02-20T22:06:13.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4487530496 bytes 2026-02-20T22:06:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4554124288 bytes 2026-02-20T22:06:13.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4616390656 bytes 2026-02-20T22:06:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4676397056 bytes 2026-02-20T22:06:13.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4718554112 bytes 2026-02-20T22:06:14.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4770226176 bytes 2026-02-20T22:06:14.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4837081088 bytes 2026-02-20T22:06:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4851456000 bytes 2026-02-20T22:06:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4917416960 bytes 2026-02-20T22:06:14.214 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4937732096 bytes 2026-02-20T22:06:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4963421184 bytes 2026-02-20T22:06:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4989810688 bytes 2026-02-20T22:06:14.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 5044930560 bytes 2026-02-20T22:06:14.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 5123812352 bytes 2026-02-20T22:06:14.454 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 5142714368 bytes 2026-02-20T22:06:14.510 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 5206279168 bytes 2026-02-20T22:06:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 5224812544 bytes 2026-02-20T22:06:14.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5267611648 bytes 2026-02-20T22:06:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5302701056 bytes 2026-02-20T22:06:14.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5326270464 bytes 2026-02-20T22:06:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5384937472 bytes 2026-02-20T22:06:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5402407936 bytes 2026-02-20T22:06:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5445923840 bytes 2026-02-20T22:06:14.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5487503360 bytes 2026-02-20T22:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5521320960 bytes 2026-02-20T22:06:14.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5523022848 bytes 2026-02-20T22:06:15.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5579092992 bytes 2026-02-20T22:06:15.060 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5654724608 bytes 2026-02-20T22:06:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5703142400 bytes 2026-02-20T22:06:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5786812416 bytes 2026-02-20T22:06:15.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5819532288 bytes 2026-02-20T22:06:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5870086144 bytes 2026-02-20T22:06:15.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5903894528 bytes 2026-02-20T22:06:15.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5938141184 bytes 2026-02-20T22:06:15.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5953474560 bytes 2026-02-20T22:06:15.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5996539904 bytes 2026-02-20T22:06:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 6055496704 bytes 2026-02-20T22:06:15.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 6074734592 bytes 2026-02-20T22:06:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 6110559232 bytes 2026-02-20T22:06:15.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 6190489600 bytes 2026-02-20T22:06:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 6258717696 bytes 2026-02-20T22:06:15.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 6307434496 bytes 2026-02-20T22:06:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 6342082560 bytes 2026-02-20T22:06:15.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 6412896256 bytes 2026-02-20T22:06:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 6432219136 bytes 2026-02-20T22:06:15.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6469147648 bytes 2026-02-20T22:06:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6515316736 bytes 2026-02-20T22:06:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6563637248 bytes 2026-02-20T22:06:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6616741888 bytes 2026-02-20T22:06:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6676492288 bytes 2026-02-20T22:06:16.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6716641280 bytes 2026-02-20T22:06:16.244 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6742400000 bytes 2026-02-20T22:06:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6803852288 bytes 2026-02-20T22:06:16.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6879534080 bytes 2026-02-20T22:06:16.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6962693120 bytes 2026-02-20T22:06:16.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6998923264 bytes 2026-02-20T22:06:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 7023286272 bytes 2026-02-20T22:06:16.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 7077359616 bytes 2026-02-20T22:06:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 7113574400 bytes 2026-02-20T22:06:16.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 7154592768 bytes 2026-02-20T22:06:16.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 7179515904 bytes 2026-02-20T22:06:16.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 7201200128 bytes 2026-02-20T22:06:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 7207542784 bytes 2026-02-20T22:06:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T22:06:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_1 2026-02-20T22:06:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T22:06:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b0e22c4e26 2026-02-20T22:06:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_1 present 10b0e22c4e26 2026-02-20T22:06:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b0e22c4e26 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_1 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T22:06:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b0e22c4e26 2026-02-20T22:06:16.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b0e22c4e26 2026-02-20T22:06:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b0e22c4e26 -o 10b0e22c4e26 = 10b0e22c4e26 2026-02-20T22:06:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:06:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:06:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_1 2026-02-20T22:06:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:16.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2026-02-20T22:06:23.643 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5370124689686967) > weight (0.25) 2026-02-20T22:06:23.644 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.9456708658841783) > weight (0.25) 2026-02-20T22:06:23.644 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8316792980764673) > weight (0.25) 2026-02-20T22:06:23.644 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.5292638439578145) > weight (0.25) 2026-02-20T22:06:23.644 INFO:tasks.rbd_mirror_thrash:waiting for 81.7021198160077 secs before thrashing 2026-02-20T22:06:24.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:06:24.089+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61e680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:06:24.097 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:06:24.089+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7d900 [3/2934a2ff-785b-412e-b7a1-904a2fa7c062] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:06:47.757 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:06:47.752+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b34f400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=185 2026-02-20T22:06:47.757 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:06:47.752+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b34f400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=188, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=b9, last_copied_object_number=128, snap_seqs={185=18446744073709551614}] 2026-02-20T22:06:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:06:47.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-20T22:06:47.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:47.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:47.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:47.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:06:47.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:47.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:47.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:47.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T22:06:47.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:47.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:47.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_1 2026-02-20T22:06:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T22:06:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:47.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-20T22:06:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:06:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:47.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-20T22:06:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:06:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T22:06:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:47.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_1 2026-02-20T22:06:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T22:06:47.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-20T22:06:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:06:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:06:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_1 2026-02-20T22:06:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2026-02-20T22:06:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-20T22:06:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:06:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:06:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_1 2026-02-20T22:06:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_1 2026-02-20T22:06:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-20T22:06:47.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-20T22:06:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:06:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_2 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_2 512M 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:06:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:06:48.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:06:48.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_2 2026-02-20T22:06:48.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_2 2026-02-20T22:06:48.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:48.181 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:48.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:06:48.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:06:48.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 snapshot 2026-02-20T22:06:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:06:48.782 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-02-20T22:06:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_2': 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 1901b5d4d6dd 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.1901b5d4d6dd 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:06:48 2026 2026-02-20T22:06:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:06:48 2026 2026-02-20T22:06:48.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:06:48 2026 2026-02-20T22:06:48.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:06:48.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:06:48.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 7d99ee4b-d7dc-4919-b507-456ad7576513 2026-02-20T22:06:48.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:06:48.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-20T22:06:48.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:48.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:48.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:06:48.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:06:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=5 2026-02-20T22:06:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:06:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:06:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 72658944 bytes 2026-02-20T22:06:49.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 143921152 bytes 2026-02-20T22:06:49.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 166235136 bytes 2026-02-20T22:06:49.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 207804416 bytes 2026-02-20T22:06:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 291892224 bytes 2026-02-20T22:06:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 338334720 bytes 2026-02-20T22:06:49.273 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 380886016 bytes 2026-02-20T22:06:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 420609024 bytes 2026-02-20T22:06:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 469794816 bytes 2026-02-20T22:06:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 492865536 bytes 2026-02-20T22:06:49.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 547673088 bytes 2026-02-20T22:06:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 565057536 bytes 2026-02-20T22:06:49.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 613970944 bytes 2026-02-20T22:06:49.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 641694720 bytes 2026-02-20T22:06:49.648 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 694920192 bytes 2026-02-20T22:06:49.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 752945152 bytes 2026-02-20T22:06:49.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 796577792 bytes 2026-02-20T22:06:49.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 829313024 bytes 2026-02-20T22:06:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 898617344 bytes 2026-02-20T22:06:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 918918144 bytes 2026-02-20T22:06:49.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 954692608 bytes 2026-02-20T22:06:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 988600320 bytes 2026-02-20T22:06:50.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 989212672 bytes 2026-02-20T22:06:50.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1034597376 bytes 2026-02-20T22:06:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1095599104 bytes 2026-02-20T22:06:50.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1134166016 bytes 2026-02-20T22:06:50.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1160936448 bytes 2026-02-20T22:06:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1192859648 bytes 2026-02-20T22:06:50.291 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1221534720 bytes 2026-02-20T22:06:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1255111680 bytes 2026-02-20T22:06:50.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1300386816 bytes 2026-02-20T22:06:50.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1325948928 bytes 2026-02-20T22:06:50.468 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1358208000 bytes 2026-02-20T22:06:50.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1386943488 bytes 2026-02-20T22:06:50.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1445706752 bytes 2026-02-20T22:06:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1486184448 bytes 2026-02-20T22:06:50.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1491243008 bytes 2026-02-20T22:06:50.694 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1526854656 bytes 2026-02-20T22:06:50.752 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1587991552 bytes 2026-02-20T22:06:50.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1629825024 bytes 2026-02-20T22:06:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1675052032 bytes 2026-02-20T22:06:50.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1734065152 bytes 2026-02-20T22:06:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1824169984 bytes 2026-02-20T22:06:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1843900416 bytes 2026-02-20T22:06:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1889611776 bytes 2026-02-20T22:06:51.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1900253184 bytes 2026-02-20T22:06:51.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1993931776 bytes 2026-02-20T22:06:51.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2002205696 bytes 2026-02-20T22:06:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2036254720 bytes 2026-02-20T22:06:51.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2130852864 bytes 2026-02-20T22:06:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2142377984 bytes 2026-02-20T22:06:51.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2177716224 bytes 2026-02-20T22:06:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2202505216 bytes 2026-02-20T22:06:51.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2211090432 bytes 2026-02-20T22:06:51.511 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2241750016 bytes 2026-02-20T22:06:51.568 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2325224448 bytes 2026-02-20T22:06:51.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2364633088 bytes 2026-02-20T22:06:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2394229760 bytes 2026-02-20T22:06:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2409909248 bytes 2026-02-20T22:06:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2433274880 bytes 2026-02-20T22:06:51.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2476272640 bytes 2026-02-20T22:06:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2563970048 bytes 2026-02-20T22:06:51.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2585733120 bytes 2026-02-20T22:06:51.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2626249728 bytes 2026-02-20T22:06:51.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2650405888 bytes 2026-02-20T22:06:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2692051968 bytes 2026-02-20T22:06:52.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2751862784 bytes 2026-02-20T22:06:52.135 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2795488256 bytes 2026-02-20T22:06:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2844022784 bytes 2026-02-20T22:06:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2933431296 bytes 2026-02-20T22:06:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2987286528 bytes 2026-02-20T22:06:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3004010496 bytes 2026-02-20T22:06:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3023578112 bytes 2026-02-20T22:06:52.430 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3064843264 bytes 2026-02-20T22:06:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3103704064 bytes 2026-02-20T22:06:52.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3155454976 bytes 2026-02-20T22:06:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3199451136 bytes 2026-02-20T22:06:52.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3239782400 bytes 2026-02-20T22:06:52.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3284933632 bytes 2026-02-20T22:06:52.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3345034240 bytes 2026-02-20T22:06:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3417953280 bytes 2026-02-20T22:06:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3500734464 bytes 2026-02-20T22:06:52.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3539026944 bytes 2026-02-20T22:06:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3564872704 bytes 2026-02-20T22:06:53.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3648703488 bytes 2026-02-20T22:06:53.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3676350464 bytes 2026-02-20T22:06:53.083 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3706760192 bytes 2026-02-20T22:06:53.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3794422784 bytes 2026-02-20T22:06:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3845021696 bytes 2026-02-20T22:06:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3852740608 bytes 2026-02-20T22:06:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3928725504 bytes 2026-02-20T22:06:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3934962688 bytes 2026-02-20T22:06:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3985036288 bytes 2026-02-20T22:06:53.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4091458560 bytes 2026-02-20T22:06:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4135312384 bytes 2026-02-20T22:06:53.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4165468160 bytes 2026-02-20T22:06:53.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4210973696 bytes 2026-02-20T22:06:53.642 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4226925568 bytes 2026-02-20T22:06:53.684 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4253638656 bytes 2026-02-20T22:06:53.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4287625216 bytes 2026-02-20T22:06:53.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4358144000 bytes 2026-02-20T22:06:53.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_2 2026-02-20T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:53.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b038c6746a 2026-02-20T22:06:53.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:06:53.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_2 present 10b038c6746a 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b038c6746a 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_2 2026-02-20T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:06:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b038c6746a 2026-02-20T22:06:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b038c6746a 2026-02-20T22:06:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b038c6746a -o 10b038c6746a = 10b038c6746a 2026-02-20T22:06:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:06:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:06:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:06:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_2 2026-02-20T22:06:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:06:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:06:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:06:53.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2026-02-20T22:06:55.779 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5723749253972155) > weight (0.25) 2026-02-20T22:06:55.780 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9547764786803737) > weight (0.25) 2026-02-20T22:06:55.780 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.35951196855083434) > weight (0.25) 2026-02-20T22:06:55.780 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.950660041019287) > weight (0.25) 2026-02-20T22:06:55.780 INFO:tasks.rbd_mirror_thrash:waiting for 74.9689834782371 secs before thrashing 2026-02-20T22:07:24.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:07:24.092+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ed9d0680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:07:24.097 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:07:24.092+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0edfe6f00 [3/7d99ee4b-d7dc-4919-b507-456ad7576513] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:07:24.753 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:07:24.748+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b049000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=186 2026-02-20T22:07:24.753 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:07:24.748+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b049000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=189, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=ba, last_copied_object_number=128, snap_seqs={186=18446744073709551614}] 2026-02-20T22:07:24.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:07:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-20T22:07:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:07:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:07:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:07:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:07:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:07:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:07:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T22:07:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:07:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:07:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_2 2026-02-20T22:07:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T22:07:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:07:24.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T22:07:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:07:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:07:24.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:07:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:07:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_2 2026-02-20T22:07:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T22:07:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:07:24.917 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T22:07:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:07:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:07:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:07:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:07:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_2 2026-02-20T22:07:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:07:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:07:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2026-02-20T22:07:24.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-20T22:07:24.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:07:24.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:24.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:07:24.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:07:24.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:07:24.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_2 2026-02-20T22:07:24.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:07:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_2 2026-02-20T22:07:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-20T22:07:24.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-20T22:07:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:07:25.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_3 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_3 512M 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:07:25.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_3 2026-02-20T22:07:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_3 2026-02-20T22:07:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:07:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:07:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:07:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 snapshot 2026-02-20T22:07:25.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:07:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-02-20T22:07:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_3': 2026-02-20T22:07:25.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:07:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:07:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:07:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 1919236e5305 2026-02-20T22:07:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.1919236e5305 2026-02-20T22:07:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:07:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:07:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:07:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:07:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:07:25 2026 2026-02-20T22:07:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:07:25 2026 2026-02-20T22:07:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:07:25 2026 2026-02-20T22:07:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:07:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:07:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: af2c4741-c758-4df5-88ef-e24e8343abff 2026-02-20T22:07:25.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:07:25.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-20T22:07:25.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:07:25.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:25.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:07:25.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:07:25.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T22:07:25.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:07:25.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:07:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 22649856 bytes 2026-02-20T22:07:25.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 31708160 bytes 2026-02-20T22:07:26.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 87961600 bytes 2026-02-20T22:07:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 140797952 bytes 2026-02-20T22:07:26.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 177116160 bytes 2026-02-20T22:07:26.194 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 220163072 bytes 2026-02-20T22:07:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 275969024 bytes 2026-02-20T22:07:26.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 326226944 bytes 2026-02-20T22:07:26.344 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 370404352 bytes 2026-02-20T22:07:26.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 418168832 bytes 2026-02-20T22:07:26.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 430759936 bytes 2026-02-20T22:07:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 481075200 bytes 2026-02-20T22:07:26.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 504986624 bytes 2026-02-20T22:07:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 573701120 bytes 2026-02-20T22:07:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 658438144 bytes 2026-02-20T22:07:26.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 775836672 bytes 2026-02-20T22:07:26.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 800295936 bytes 2026-02-20T22:07:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 915548160 bytes 2026-02-20T22:07:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 972963840 bytes 2026-02-20T22:07:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1027459072 bytes 2026-02-20T22:07:26.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1049171968 bytes 2026-02-20T22:07:27.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1085614080 bytes 2026-02-20T22:07:27.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1109861376 bytes 2026-02-20T22:07:27.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1137942528 bytes 2026-02-20T22:07:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1173406720 bytes 2026-02-20T22:07:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1190871040 bytes 2026-02-20T22:07:27.249 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1229386752 bytes 2026-02-20T22:07:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1248251904 bytes 2026-02-20T22:07:27.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1368119296 bytes 2026-02-20T22:07:27.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1368732672 bytes 2026-02-20T22:07:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1410694144 bytes 2026-02-20T22:07:27.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1456918528 bytes 2026-02-20T22:07:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1485902848 bytes 2026-02-20T22:07:27.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1486754816 bytes 2026-02-20T22:07:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1534183424 bytes 2026-02-20T22:07:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1580230656 bytes 2026-02-20T22:07:27.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1605356544 bytes 2026-02-20T22:07:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1650026496 bytes 2026-02-20T22:07:27.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1672877056 bytes 2026-02-20T22:07:27.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1720200192 bytes 2026-02-20T22:07:27.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1828207616 bytes 2026-02-20T22:07:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1879992320 bytes 2026-02-20T22:07:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1903457280 bytes 2026-02-20T22:07:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1975784448 bytes 2026-02-20T22:07:28.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1982077952 bytes 2026-02-20T22:07:28.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2035994624 bytes 2026-02-20T22:07:28.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2087913472 bytes 2026-02-20T22:07:28.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2098023424 bytes 2026-02-20T22:07:28.297 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2130422784 bytes 2026-02-20T22:07:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2197004288 bytes 2026-02-20T22:07:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2213815296 bytes 2026-02-20T22:07:28.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2255700992 bytes 2026-02-20T22:07:28.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2257437696 bytes 2026-02-20T22:07:28.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2285032448 bytes 2026-02-20T22:07:28.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2319250432 bytes 2026-02-20T22:07:28.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2352445440 bytes 2026-02-20T22:07:28.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2386571264 bytes 2026-02-20T22:07:28.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2423211008 bytes 2026-02-20T22:07:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2435896320 bytes 2026-02-20T22:07:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2495818752 bytes 2026-02-20T22:07:28.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2531960832 bytes 2026-02-20T22:07:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2601578496 bytes 2026-02-20T22:07:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2689117184 bytes 2026-02-20T22:07:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2751762432 bytes 2026-02-20T22:07:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2808978432 bytes 2026-02-20T22:07:29.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2809577472 bytes 2026-02-20T22:07:29.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2815724544 bytes 2026-02-20T22:07:29.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2839805952 bytes 2026-02-20T22:07:29.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2867396608 bytes 2026-02-20T22:07:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2901252096 bytes 2026-02-20T22:07:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2992479232 bytes 2026-02-20T22:07:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3059338240 bytes 2026-02-20T22:07:29.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3070729216 bytes 2026-02-20T22:07:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3151498240 bytes 2026-02-20T22:07:29.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3176530944 bytes 2026-02-20T22:07:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3211622400 bytes 2026-02-20T22:07:29.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3283372032 bytes 2026-02-20T22:07:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3359943680 bytes 2026-02-20T22:07:29.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3412018176 bytes 2026-02-20T22:07:29.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3426480128 bytes 2026-02-20T22:07:30.052 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3474782208 bytes 2026-02-20T22:07:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3542390784 bytes 2026-02-20T22:07:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3599762432 bytes 2026-02-20T22:07:30.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3614389248 bytes 2026-02-20T22:07:30.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3634804736 bytes 2026-02-20T22:07:30.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3673063424 bytes 2026-02-20T22:07:30.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3698750464 bytes 2026-02-20T22:07:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3731235840 bytes 2026-02-20T22:07:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3779045376 bytes 2026-02-20T22:07:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3798871040 bytes 2026-02-20T22:07:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3891844096 bytes 2026-02-20T22:07:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3900510208 bytes 2026-02-20T22:07:30.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3978656768 bytes 2026-02-20T22:07:30.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4020136960 bytes 2026-02-20T22:07:30.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4051085312 bytes 2026-02-20T22:07:30.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4091652096 bytes 2026-02-20T22:07:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4106058752 bytes 2026-02-20T22:07:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4147724288 bytes 2026-02-20T22:07:30.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4212428800 bytes 2026-02-20T22:07:30.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4254204928 bytes 2026-02-20T22:07:30.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4316261376 bytes 2026-02-20T22:07:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4341036032 bytes 2026-02-20T22:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4375170048 bytes 2026-02-20T22:07:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4399337472 bytes 2026-02-20T22:07:31.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4432911360 bytes 2026-02-20T22:07:31.220 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4459940864 bytes 2026-02-20T22:07:31.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4552727552 bytes 2026-02-20T22:07:31.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4566085632 bytes 2026-02-20T22:07:31.361 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4591886336 bytes 2026-02-20T22:07:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4614094848 bytes 2026-02-20T22:07:31.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4646184960 bytes 2026-02-20T22:07:31.534 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4790254592 bytes 2026-02-20T22:07:31.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4798171136 bytes 2026-02-20T22:07:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4847544320 bytes 2026-02-20T22:07:31.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4867051520 bytes 2026-02-20T22:07:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4935639040 bytes 2026-02-20T22:07:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4984219648 bytes 2026-02-20T22:07:31.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5022654464 bytes 2026-02-20T22:07:31.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5082100736 bytes 2026-02-20T22:07:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5086586880 bytes 2026-02-20T22:07:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5102800896 bytes 2026-02-20T22:07:31.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5150947328 bytes 2026-02-20T22:07:32.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5185775616 bytes 2026-02-20T22:07:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5211450368 bytes 2026-02-20T22:07:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5256655872 bytes 2026-02-20T22:07:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5361363968 bytes 2026-02-20T22:07:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5441356800 bytes 2026-02-20T22:07:32.302 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5451251712 bytes 2026-02-20T22:07:32.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5462733824 bytes 2026-02-20T22:07:32.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5475722240 bytes 2026-02-20T22:07:32.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5549211648 bytes 2026-02-20T22:07:32.467 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5560754176 bytes 2026-02-20T22:07:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5600671744 bytes 2026-02-20T22:07:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5635190784 bytes 2026-02-20T22:07:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5692964864 bytes 2026-02-20T22:07:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5720088576 bytes 2026-02-20T22:07:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5732484096 bytes 2026-02-20T22:07:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5772000256 bytes 2026-02-20T22:07:32.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5802310656 bytes 2026-02-20T22:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5837077504 bytes 2026-02-20T22:07:32.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5893469184 bytes 2026-02-20T22:07:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5943171072 bytes 2026-02-20T22:07:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6000012288 bytes 2026-02-20T22:07:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6061401088 bytes 2026-02-20T22:07:33.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6123586560 bytes 2026-02-20T22:07:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6146763776 bytes 2026-02-20T22:07:33.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6177152000 bytes 2026-02-20T22:07:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6189922304 bytes 2026-02-20T22:07:33.493 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6227542016 bytes 2026-02-20T22:07:33.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6257810432 bytes 2026-02-20T22:07:33.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6277040128 bytes 2026-02-20T22:07:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6307929088 bytes 2026-02-20T22:07:33.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6344154112 bytes 2026-02-20T22:07:33.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6354956288 bytes 2026-02-20T22:07:33.749 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6387562496 bytes 2026-02-20T22:07:33.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6411473920 bytes 2026-02-20T22:07:33.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6475476992 bytes 2026-02-20T22:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6491595776 bytes 2026-02-20T22:07:33.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6559502336 bytes 2026-02-20T22:07:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6575826944 bytes 2026-02-20T22:07:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6579476480 bytes 2026-02-20T22:07:34.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6680212480 bytes 2026-02-20T22:07:34.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6705074176 bytes 2026-02-20T22:07:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6739598336 bytes 2026-02-20T22:07:34.228 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6787451904 bytes 2026-02-20T22:07:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6869270528 bytes 2026-02-20T22:07:34.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6919229440 bytes 2026-02-20T22:07:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6926304256 bytes 2026-02-20T22:07:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 6969131008 bytes 2026-02-20T22:07:34.479 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7025059840 bytes 2026-02-20T22:07:34.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7046505472 bytes 2026-02-20T22:07:34.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7088468992 bytes 2026-02-20T22:07:34.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7099549696 bytes 2026-02-20T22:07:34.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7163652096 bytes 2026-02-20T22:07:34.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7227773952 bytes 2026-02-20T22:07:34.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7283836928 bytes 2026-02-20T22:07:34.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7329492992 bytes 2026-02-20T22:07:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:07:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:07:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:07:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:07:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2026-02-20T22:07:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:07:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:34.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_3 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T22:07:34.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:07:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b03501deae 2026-02-20T22:07:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:07:34.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:07:34.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_3 present 10b03501deae 2026-02-20T22:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b03501deae 2026-02-20T22:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T22:07:34.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:07:34.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:07:34.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_3 2026-02-20T22:07:34.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T22:07:34.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:07:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b03501deae 2026-02-20T22:07:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b03501deae 2026-02-20T22:07:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b03501deae -o 10b03501deae = 10b03501deae 2026-02-20T22:07:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:07:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:07:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:07:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_3 2026-02-20T22:07:34.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:07:34.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:07:34.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:07:34.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2026-02-20T22:07:45.347 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.795873505848522) > weight (0.25) 2026-02-20T22:07:45.347 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.42939316383735904) > weight (0.25) 2026-02-20T22:07:45.348 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.33199801253153416) > weight (0.25) 2026-02-20T22:07:45.348 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-20T22:07:45.348 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-20T22:07:45.348 INFO:tasks.rbd_mirror_thrash:waiting for 0.6715581219607913 secs before reviving daemons 2026-02-20T22:07:45.349 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:07:45.343+0000 7fb94d18f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 47297) UID: 1000 2026-02-20T22:07:46.020 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-20T22:07:46.020 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:07:46.021 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-20T22:07:46.021 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-20T22:07:46.021 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-20T22:07:46.023 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-20T22:07:46.023 INFO:tasks.rbd_mirror_thrash:waiting for 68.7357307297863 secs before thrashing 2026-02-20T22:07:54.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:07:54.091+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61e340 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:07:54.097 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:07:54.091+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7d900 [3/af2c4741-c758-4df5-88ef-e24e8343abff] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:08:05.757 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:08:05.751+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b9c5c00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=187 2026-02-20T22:08:05.757 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:08:05.751+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b9c5c00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=190, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=bb, last_copied_object_number=128, snap_seqs={187=18446744073709551614}] 2026-02-20T22:08:05.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:08:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-20T22:08:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:08:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:08:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:08:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:08:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:08:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:08:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T22:08:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:08:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:08:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_3 2026-02-20T22:08:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T22:08:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:08:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-20T22:08:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:08:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:08:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:08:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-20T22:08:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:08:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:08:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:08:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:08:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:08:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:08:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T22:08:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:08:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:08:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_3 2026-02-20T22:08:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T22:08:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:08:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-20T22:08:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:08:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:08:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:08:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:08:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_3 2026-02-20T22:08:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:08:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2026-02-20T22:08:05.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-20T22:08:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:08:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:08:05.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:08:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_3 2026-02-20T22:08:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_3 2026-02-20T22:08:05.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-20T22:08:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-20T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:08:06.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:08:06.143 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_4 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_4 512M 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:08:06.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:08:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:08:06.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_4 2026-02-20T22:08:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_4 2026-02-20T22:08:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:08:06.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 snapshot 2026-02-20T22:08:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:08:06.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-02-20T22:08:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_4': 2026-02-20T22:08:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:08:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:08:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:08:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 193a637e9feb 2026-02-20T22:08:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.193a637e9feb 2026-02-20T22:08:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:08:06 2026 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:08:06 2026 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:08:06 2026 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 2650848d-6383-4dd0-97ed-8efb40f37dff 2026-02-20T22:08:06.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:08:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-20T22:08:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:08:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=6 2026-02-20T22:08:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:08:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:08:06.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 44276736 bytes 2026-02-20T22:08:07.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 94135296 bytes 2026-02-20T22:08:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 137705472 bytes 2026-02-20T22:08:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 230539264 bytes 2026-02-20T22:08:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 272033792 bytes 2026-02-20T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 295240704 bytes 2026-02-20T22:08:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 355684352 bytes 2026-02-20T22:08:07.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 417548288 bytes 2026-02-20T22:08:07.373 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 451984384 bytes 2026-02-20T22:08:07.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 501685248 bytes 2026-02-20T22:08:07.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 587983872 bytes 2026-02-20T22:08:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 682948608 bytes 2026-02-20T22:08:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 731949056 bytes 2026-02-20T22:08:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 775060480 bytes 2026-02-20T22:08:07.720 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 890200064 bytes 2026-02-20T22:08:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 917530624 bytes 2026-02-20T22:08:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 919335936 bytes 2026-02-20T22:08:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 931463168 bytes 2026-02-20T22:08:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 981940224 bytes 2026-02-20T22:08:07.947 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 1030268928 bytes 2026-02-20T22:08:07.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1084785664 bytes 2026-02-20T22:08:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1110235136 bytes 2026-02-20T22:08:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1196608512 bytes 2026-02-20T22:08:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1336667136 bytes 2026-02-20T22:08:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1389655040 bytes 2026-02-20T22:08:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1433470976 bytes 2026-02-20T22:08:08.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1462996992 bytes 2026-02-20T22:08:08.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1533440000 bytes 2026-02-20T22:08:08.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1563513856 bytes 2026-02-20T22:08:08.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1599561728 bytes 2026-02-20T22:08:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1651428352 bytes 2026-02-20T22:08:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1676726272 bytes 2026-02-20T22:08:08.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1721395200 bytes 2026-02-20T22:08:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1722917888 bytes 2026-02-20T22:08:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1785982976 bytes 2026-02-20T22:08:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1815421952 bytes 2026-02-20T22:08:08.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1877417984 bytes 2026-02-20T22:08:08.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1936287744 bytes 2026-02-20T22:08:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1977958400 bytes 2026-02-20T22:08:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 2061593600 bytes 2026-02-20T22:08:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 2164985856 bytes 2026-02-20T22:08:09.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 2184964096 bytes 2026-02-20T22:08:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 2226113536 bytes 2026-02-20T22:08:09.177 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 2260995072 bytes 2026-02-20T22:08:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 2323146752 bytes 2026-02-20T22:08:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2420836352 bytes 2026-02-20T22:08:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2450065408 bytes 2026-02-20T22:08:09.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2487240704 bytes 2026-02-20T22:08:09.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2546967552 bytes 2026-02-20T22:08:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2573746176 bytes 2026-02-20T22:08:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2574373888 bytes 2026-02-20T22:08:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2665695232 bytes 2026-02-20T22:08:09.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2692289536 bytes 2026-02-20T22:08:09.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2722173952 bytes 2026-02-20T22:08:09.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2767076352 bytes 2026-02-20T22:08:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2861366272 bytes 2026-02-20T22:08:09.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2861969408 bytes 2026-02-20T22:08:09.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2925604864 bytes 2026-02-20T22:08:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2935383040 bytes 2026-02-20T22:08:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2938650624 bytes 2026-02-20T22:08:09.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2980180992 bytes 2026-02-20T22:08:10.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 3034173440 bytes 2026-02-20T22:08:10.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 3085407232 bytes 2026-02-20T22:08:10.121 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 3085998080 bytes 2026-02-20T22:08:10.167 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 3110723584 bytes 2026-02-20T22:08:10.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 3117410304 bytes 2026-02-20T22:08:10.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3147782144 bytes 2026-02-20T22:08:10.309 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3218417664 bytes 2026-02-20T22:08:10.351 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3245186048 bytes 2026-02-20T22:08:10.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3273442304 bytes 2026-02-20T22:08:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3371337728 bytes 2026-02-20T22:08:10.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3417777152 bytes 2026-02-20T22:08:10.541 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3450006528 bytes 2026-02-20T22:08:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3527667712 bytes 2026-02-20T22:08:10.650 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3546355712 bytes 2026-02-20T22:08:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3580816384 bytes 2026-02-20T22:08:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3626092544 bytes 2026-02-20T22:08:10.750 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-20T22:08:10.750 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-20T22:08:10.750 INFO:tasks.rbd_mirror_thrash:waiting for 5.581883403607804 secs before reviving daemons 2026-02-20T22:08:10.751 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:08:10.747+0000 7f673b7fc640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 42531) UID: 1000 2026-02-20T22:08:10.795 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3692936192 bytes 2026-02-20T22:08:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3724906496 bytes 2026-02-20T22:08:10.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3762734080 bytes 2026-02-20T22:08:10.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3805778944 bytes 2026-02-20T22:08:10.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3816563712 bytes 2026-02-20T22:08:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3854906368 bytes 2026-02-20T22:08:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3914590208 bytes 2026-02-20T22:08:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3935411200 bytes 2026-02-20T22:08:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3963250688 bytes 2026-02-20T22:08:11.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 4050117632 bytes 2026-02-20T22:08:11.263 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4075319296 bytes 2026-02-20T22:08:11.303 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4100150272 bytes 2026-02-20T22:08:11.347 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4148144128 bytes 2026-02-20T22:08:11.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4200635392 bytes 2026-02-20T22:08:11.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4277683200 bytes 2026-02-20T22:08:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4325763072 bytes 2026-02-20T22:08:11.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4405795840 bytes 2026-02-20T22:08:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4426705920 bytes 2026-02-20T22:08:11.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4433473536 bytes 2026-02-20T22:08:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4486410240 bytes 2026-02-20T22:08:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4581707776 bytes 2026-02-20T22:08:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4639659008 bytes 2026-02-20T22:08:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4707104768 bytes 2026-02-20T22:08:11.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4787853312 bytes 2026-02-20T22:08:11.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4852457472 bytes 2026-02-20T22:08:12.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4887696384 bytes 2026-02-20T22:08:12.078 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4903243776 bytes 2026-02-20T22:08:12.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4945981440 bytes 2026-02-20T22:08:12.176 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 5001605120 bytes 2026-02-20T22:08:12.226 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 5040156672 bytes 2026-02-20T22:08:12.277 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 5075788800 bytes 2026-02-20T22:08:12.337 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 5172311040 bytes 2026-02-20T22:08:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 5213921280 bytes 2026-02-20T22:08:12.432 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 5236888576 bytes 2026-02-20T22:08:12.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 5267301376 bytes 2026-02-20T22:08:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5317189632 bytes 2026-02-20T22:08:12.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5393735680 bytes 2026-02-20T22:08:12.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5444678656 bytes 2026-02-20T22:08:12.672 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5463921664 bytes 2026-02-20T22:08:12.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5472244736 bytes 2026-02-20T22:08:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5514654720 bytes 2026-02-20T22:08:12.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5535040512 bytes 2026-02-20T22:08:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:08:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:08:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:08:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:08:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2026-02-20T22:08:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:08:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:12.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:08:12.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:08:12.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:08:12.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:08:12.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T22:08:12.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:08:12.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:08:12.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_4 2026-02-20T22:08:12.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T22:08:12.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b0175fa9f 2026-02-20T22:08:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:08:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:08:12.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_4 present 10b0175fa9f 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b0175fa9f 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:08:12.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_4 2026-02-20T22:08:12.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T22:08:12.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:08:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b0175fa9f 2026-02-20T22:08:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b0175fa9f 2026-02-20T22:08:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b0175fa9f -o 10b0175fa9f = 10b0175fa9f 2026-02-20T22:08:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:08:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:08:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:08:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_4 2026-02-20T22:08:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2026-02-20T22:08:16.333 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-20T22:08:16.334 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:08:16.334 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-20T22:08:16.334 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T22:08:16.334 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T22:08:16.336 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T22:08:16.336 INFO:tasks.rbd_mirror_thrash:waiting for 84.8997359804504 secs before thrashing 2026-02-20T22:08:24.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:08:24.090+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61e9c0 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:08:24.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:08:24.090+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50f400 [3/2650848d-6383-4dd0-97ed-8efb40f37dff] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:08:43.759 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:08:43.754+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7bbfb000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=188 2026-02-20T22:08:43.759 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:08:43.754+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7bbfb000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=191, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=bc, last_copied_object_number=128, snap_seqs={188=18446744073709551614}] 2026-02-20T22:08:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:08:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-20T22:08:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:08:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:08:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:08:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:08:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:08:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T22:08:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:08:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:08:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_4 2026-02-20T22:08:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T22:08:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:08:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-20T22:08:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:08:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:08:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:08:43.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-20T22:08:43.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:08:43.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_4 2026-02-20T22:08:43.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T22:08:43.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:08:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-20T22:08:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:08:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:08:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:08:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:08:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_4 2026-02-20T22:08:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2026-02-20T22:08:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-20T22:08:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:08:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:08:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_4 2026-02-20T22:08:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_4 2026-02-20T22:08:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-20T22:08:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-20T22:08:44.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:08:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:08:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:08:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_5 2026-02-20T22:08:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_5 512M 2026-02-20T22:08:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:08:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_5 2026-02-20T22:08:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_5 2026-02-20T22:08:44.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:08:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:08:44.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 snapshot 2026-02-20T22:08:44.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:08:44.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-02-20T22:08:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_5': 2026-02-20T22:08:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:08:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:08:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:08:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 19581d83b4aa 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.19581d83b4aa 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:08:44 2026 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:08:44 2026 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:08:44 2026 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:08:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: f434c360-50b4-4608-983f-18f140bf2989 2026-02-20T22:08:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:08:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-20T22:08:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:08:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:08:44.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=11 2026-02-20T22:08:44.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:08:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:08:44.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 70586368 bytes 2026-02-20T22:08:45.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 77817856 bytes 2026-02-20T22:08:45.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 86946816 bytes 2026-02-20T22:08:45.105 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 141976576 bytes 2026-02-20T22:08:45.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 172763136 bytes 2026-02-20T22:08:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 198844416 bytes 2026-02-20T22:08:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 247962624 bytes 2026-02-20T22:08:45.300 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 309748736 bytes 2026-02-20T22:08:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 347739136 bytes 2026-02-20T22:08:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 368301056 bytes 2026-02-20T22:08:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 435876864 bytes 2026-02-20T22:08:45.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 471925760 bytes 2026-02-20T22:08:45.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 480606208 bytes 2026-02-20T22:08:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 571554816 bytes 2026-02-20T22:08:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 599038976 bytes 2026-02-20T22:08:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 667478016 bytes 2026-02-20T22:08:45.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 750555136 bytes 2026-02-20T22:08:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 790983680 bytes 2026-02-20T22:08:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 823548928 bytes 2026-02-20T22:08:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 844485632 bytes 2026-02-20T22:08:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 931971072 bytes 2026-02-20T22:08:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1014929408 bytes 2026-02-20T22:08:46.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1030363136 bytes 2026-02-20T22:08:46.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1064547328 bytes 2026-02-20T22:08:46.133 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1092961280 bytes 2026-02-20T22:08:46.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1124038656 bytes 2026-02-20T22:08:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1183967232 bytes 2026-02-20T22:08:46.279 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1210387456 bytes 2026-02-20T22:08:46.329 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1249098752 bytes 2026-02-20T22:08:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1295608832 bytes 2026-02-20T22:08:46.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1348413440 bytes 2026-02-20T22:08:46.477 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1375847424 bytes 2026-02-20T22:08:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1431943168 bytes 2026-02-20T22:08:46.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1464932352 bytes 2026-02-20T22:08:46.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1521512448 bytes 2026-02-20T22:08:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1589112832 bytes 2026-02-20T22:08:46.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1589719040 bytes 2026-02-20T22:08:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1606693888 bytes 2026-02-20T22:08:46.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1608784896 bytes 2026-02-20T22:08:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1664960512 bytes 2026-02-20T22:08:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1671541760 bytes 2026-02-20T22:08:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1709132800 bytes 2026-02-20T22:08:46.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1768755200 bytes 2026-02-20T22:08:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1771470848 bytes 2026-02-20T22:08:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1820194816 bytes 2026-02-20T22:08:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1821407232 bytes 2026-02-20T22:08:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1865180160 bytes 2026-02-20T22:08:47.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1945780224 bytes 2026-02-20T22:08:47.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1975452672 bytes 2026-02-20T22:08:47.290 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1986158592 bytes 2026-02-20T22:08:47.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2036761600 bytes 2026-02-20T22:08:47.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2038928384 bytes 2026-02-20T22:08:47.423 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2075843584 bytes 2026-02-20T22:08:47.492 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2178935808 bytes 2026-02-20T22:08:47.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2219444224 bytes 2026-02-20T22:08:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2289496064 bytes 2026-02-20T22:08:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2355524608 bytes 2026-02-20T22:08:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2419784704 bytes 2026-02-20T22:08:47.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2536972288 bytes 2026-02-20T22:08:47.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2622826496 bytes 2026-02-20T22:08:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2644998144 bytes 2026-02-20T22:08:47.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2678413312 bytes 2026-02-20T22:08:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2717188096 bytes 2026-02-20T22:08:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2791435264 bytes 2026-02-20T22:08:48.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2802784256 bytes 2026-02-20T22:08:48.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2838950912 bytes 2026-02-20T22:08:48.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2892992512 bytes 2026-02-20T22:08:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2942606336 bytes 2026-02-20T22:08:48.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3006771200 bytes 2026-02-20T22:08:48.358 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3037621248 bytes 2026-02-20T22:08:48.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3069090816 bytes 2026-02-20T22:08:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3090397184 bytes 2026-02-20T22:08:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3110782976 bytes 2026-02-20T22:08:48.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3145395200 bytes 2026-02-20T22:08:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3187247104 bytes 2026-02-20T22:08:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3216508928 bytes 2026-02-20T22:08:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3256953856 bytes 2026-02-20T22:08:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3349446656 bytes 2026-02-20T22:08:48.774 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3354081280 bytes 2026-02-20T22:08:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3384932352 bytes 2026-02-20T22:08:48.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3448964096 bytes 2026-02-20T22:08:48.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3508538368 bytes 2026-02-20T22:08:48.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3560639488 bytes 2026-02-20T22:08:49.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3611196416 bytes 2026-02-20T22:08:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3650788352 bytes 2026-02-20T22:08:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3712294912 bytes 2026-02-20T22:08:49.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3759995904 bytes 2026-02-20T22:08:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3791284224 bytes 2026-02-20T22:08:49.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3825940480 bytes 2026-02-20T22:08:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3840560128 bytes 2026-02-20T22:08:49.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3941517312 bytes 2026-02-20T22:08:49.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4017680384 bytes 2026-02-20T22:08:49.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4048162816 bytes 2026-02-20T22:08:49.536 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4113275904 bytes 2026-02-20T22:08:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4124236800 bytes 2026-02-20T22:08:49.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4189940736 bytes 2026-02-20T22:08:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4251392000 bytes 2026-02-20T22:08:49.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4289331200 bytes 2026-02-20T22:08:49.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4373886976 bytes 2026-02-20T22:08:49.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4457143296 bytes 2026-02-20T22:08:49.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4483340288 bytes 2026-02-20T22:08:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4499146752 bytes 2026-02-20T22:08:50.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4556455936 bytes 2026-02-20T22:08:50.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4601905152 bytes 2026-02-20T22:08:50.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4621915136 bytes 2026-02-20T22:08:50.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4666625024 bytes 2026-02-20T22:08:50.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4667223040 bytes 2026-02-20T22:08:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4674927616 bytes 2026-02-20T22:08:50.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4682488832 bytes 2026-02-20T22:08:50.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4685918208 bytes 2026-02-20T22:08:50.323 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4707985408 bytes 2026-02-20T22:08:50.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4789936128 bytes 2026-02-20T22:08:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4829671424 bytes 2026-02-20T22:08:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4878042112 bytes 2026-02-20T22:08:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4900152320 bytes 2026-02-20T22:08:50.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4933199872 bytes 2026-02-20T22:08:50.624 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4975026176 bytes 2026-02-20T22:08:50.674 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5019695104 bytes 2026-02-20T22:08:50.729 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5080116224 bytes 2026-02-20T22:08:50.780 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5120229376 bytes 2026-02-20T22:08:50.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5120843776 bytes 2026-02-20T22:08:50.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5167717376 bytes 2026-02-20T22:08:50.900 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5182953472 bytes 2026-02-20T22:08:50.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5237132288 bytes 2026-02-20T22:08:50.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5262568448 bytes 2026-02-20T22:08:51.048 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5329852416 bytes 2026-02-20T22:08:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5355343872 bytes 2026-02-20T22:08:51.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5402083328 bytes 2026-02-20T22:08:51.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5450706944 bytes 2026-02-20T22:08:51.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5505596416 bytes 2026-02-20T22:08:51.304 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5568893952 bytes 2026-02-20T22:08:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5577731072 bytes 2026-02-20T22:08:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5659479040 bytes 2026-02-20T22:08:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5697385472 bytes 2026-02-20T22:08:51.497 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5742997504 bytes 2026-02-20T22:08:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5762190336 bytes 2026-02-20T22:08:51.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5762801664 bytes 2026-02-20T22:08:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5819119616 bytes 2026-02-20T22:08:51.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5863379968 bytes 2026-02-20T22:08:51.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5878459392 bytes 2026-02-20T22:08:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5913901056 bytes 2026-02-20T22:08:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5923621888 bytes 2026-02-20T22:08:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5924257792 bytes 2026-02-20T22:08:51.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6005353472 bytes 2026-02-20T22:08:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6060234752 bytes 2026-02-20T22:08:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6075320320 bytes 2026-02-20T22:08:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6142332928 bytes 2026-02-20T22:08:52.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6188428288 bytes 2026-02-20T22:08:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6215061504 bytes 2026-02-20T22:08:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6235864064 bytes 2026-02-20T22:08:52.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6321280000 bytes 2026-02-20T22:08:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6423202816 bytes 2026-02-20T22:08:52.372 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6502063104 bytes 2026-02-20T22:08:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6562016256 bytes 2026-02-20T22:08:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6626319360 bytes 2026-02-20T22:08:52.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6649926656 bytes 2026-02-20T22:08:52.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6702764032 bytes 2026-02-20T22:08:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6703382528 bytes 2026-02-20T22:08:52.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6712522752 bytes 2026-02-20T22:08:52.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6751112192 bytes 2026-02-20T22:08:52.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6829798400 bytes 2026-02-20T22:08:52.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6853150720 bytes 2026-02-20T22:08:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6902663168 bytes 2026-02-20T22:08:52.896 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6935201792 bytes 2026-02-20T22:08:52.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6989701120 bytes 2026-02-20T22:08:53.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7055907840 bytes 2026-02-20T22:08:53.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7153955840 bytes 2026-02-20T22:08:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7222396928 bytes 2026-02-20T22:08:53.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7223007232 bytes 2026-02-20T22:08:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7249662976 bytes 2026-02-20T22:08:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7304828928 bytes 2026-02-20T22:08:53.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7313671168 bytes 2026-02-20T22:08:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7330535424 bytes 2026-02-20T22:08:53.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7416433664 bytes 2026-02-20T22:08:53.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7430263808 bytes 2026-02-20T22:08:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7471919104 bytes 2026-02-20T22:08:53.517 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7482347520 bytes 2026-02-20T22:08:53.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7522196480 bytes 2026-02-20T22:08:53.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7567251456 bytes 2026-02-20T22:08:53.663 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7623337984 bytes 2026-02-20T22:08:53.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7655117824 bytes 2026-02-20T22:08:53.762 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7709897728 bytes 2026-02-20T22:08:53.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7751108608 bytes 2026-02-20T22:08:53.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7776476160 bytes 2026-02-20T22:08:53.903 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7828106240 bytes 2026-02-20T22:08:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7846374400 bytes 2026-02-20T22:08:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7914516480 bytes 2026-02-20T22:08:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7939288064 bytes 2026-02-20T22:08:54.093 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7986832384 bytes 2026-02-20T22:08:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8002275328 bytes 2026-02-20T22:08:54.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8104040448 bytes 2026-02-20T22:08:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8159117312 bytes 2026-02-20T22:08:54.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8177705984 bytes 2026-02-20T22:08:54.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8221853696 bytes 2026-02-20T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8256176128 bytes 2026-02-20T22:08:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8291065856 bytes 2026-02-20T22:08:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8379617280 bytes 2026-02-20T22:08:54.543 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8380274688 bytes 2026-02-20T22:08:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8405750784 bytes 2026-02-20T22:08:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8452786176 bytes 2026-02-20T22:08:54.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8520664064 bytes 2026-02-20T22:08:54.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8558176256 bytes 2026-02-20T22:08:54.760 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9771359292575484) > weight (0.25) 2026-02-20T22:08:54.760 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-20T22:08:54.761 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-20T22:08:54.761 INFO:tasks.rbd_mirror_thrash:waiting for 5.777442106703638 secs before reviving daemons 2026-02-20T22:08:54.761 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:08:54.754+0000 7f4aa6514640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 36500) UID: 1000 2026-02-20T22:08:54.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8605704192 bytes 2026-02-20T22:08:54.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8646120448 bytes 2026-02-20T22:08:54.918 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8717404160 bytes 2026-02-20T22:08:54.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8754768896 bytes 2026-02-20T22:08:55.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8824944640 bytes 2026-02-20T22:08:55.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8869816320 bytes 2026-02-20T22:08:55.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8893458432 bytes 2026-02-20T22:08:55.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8928241664 bytes 2026-02-20T22:08:55.204 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8977885184 bytes 2026-02-20T22:08:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9019426816 bytes 2026-02-20T22:08:55.298 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9069827072 bytes 2026-02-20T22:08:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9129358336 bytes 2026-02-20T22:08:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9230115840 bytes 2026-02-20T22:08:55.476 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9266237440 bytes 2026-02-20T22:08:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9309947904 bytes 2026-02-20T22:08:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9393582080 bytes 2026-02-20T22:08:55.620 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9394216960 bytes 2026-02-20T22:08:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9435090944 bytes 2026-02-20T22:08:55.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9466807296 bytes 2026-02-20T22:08:55.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9512341504 bytes 2026-02-20T22:08:55.812 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9570362368 bytes 2026-02-20T22:08:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:08:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:08:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:08:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:08:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_5 2026-02-20T22:08:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T22:08:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:08:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b0aaa6d854 2026-02-20T22:08:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:08:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:08:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_5 present 10b0aaa6d854 2026-02-20T22:08:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:08:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:08:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:08:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b0aaa6d854 2026-02-20T22:08:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:08:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:08:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T22:08:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:08:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:08:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_5 2026-02-20T22:08:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T22:08:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:08:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b0aaa6d854 2026-02-20T22:08:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b0aaa6d854 2026-02-20T22:08:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b0aaa6d854 -o 10b0aaa6d854 = 10b0aaa6d854 2026-02-20T22:08:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:08:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:08:55.981 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:08:55.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_5 2026-02-20T22:08:55.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:08:55.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:08:55.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:08:55.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2026-02-20T22:09:00.539 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-20T22:09:00.539 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:09:00.540 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-20T22:09:00.540 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T22:09:00.540 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T22:09:00.542 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T22:09:00.543 INFO:tasks.rbd_mirror_thrash:waiting for 63.53466966009232 secs before thrashing 2026-02-20T22:09:24.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:09:24.089+0000 7f147e265640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0edd0e680 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:09:24.097 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:09:24.089+0000 7f1484a72640 -1 rbd::mirror::ImageReplayer: 0x55f0ee50f400 [3/f434c360-50b4-4608-983f-18f140bf2989] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:09:26.768 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:09:26.761+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7da3e000 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=189 2026-02-20T22:09:26.769 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:09:26.761+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7da3e000 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=192, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=bd, last_copied_object_number=128, snap_seqs={189=18446744073709551614}] 2026-02-20T22:09:26.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:09:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-20T22:09:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:09:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:09:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:09:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:09:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:09:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:09:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T22:09:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:09:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:09:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_5 2026-02-20T22:09:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T22:09:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:09:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-20T22:09:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:09:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:09:26.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:09:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-20T22:09:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:09:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_5 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T22:09:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:09:26.935 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-20T22:09:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:09:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:09:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:09:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:09:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_5 2026-02-20T22:09:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:09:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:09:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2026-02-20T22:09:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-20T22:09:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:09:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:09:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:09:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:09:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_5 2026-02-20T22:09:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:09:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_5 2026-02-20T22:09:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-20T22:09:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-20T22:09:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:09:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:09:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_6 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_6 512M 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:09:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:09:27.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:09:27.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_6 2026-02-20T22:09:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_6 2026-02-20T22:09:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:09:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:09:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:09:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 snapshot 2026-02-20T22:09:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:09:27.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-02-20T22:09:27.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_6': 2026-02-20T22:09:27.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:09:27.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:09:27.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 19792161a993 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.19792161a993 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:09:27 2026 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:09:27 2026 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:09:27 2026 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:09:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:09:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 51b51193-d420-435d-bd9f-2a836d56de53 2026-02-20T22:09:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:09:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-20T22:09:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:09:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:09:27.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:09:27.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=11 2026-02-20T22:09:27.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:09:27.819 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:09:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 73944064 bytes 2026-02-20T22:09:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 99522560 bytes 2026-02-20T22:09:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 163290112 bytes 2026-02-20T22:09:28.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 214256640 bytes 2026-02-20T22:09:28.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 254817280 bytes 2026-02-20T22:09:28.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 282524672 bytes 2026-02-20T22:09:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 303126528 bytes 2026-02-20T22:09:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 334859264 bytes 2026-02-20T22:09:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 373344256 bytes 2026-02-20T22:09:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 399048704 bytes 2026-02-20T22:09:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 468686848 bytes 2026-02-20T22:09:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 492259328 bytes 2026-02-20T22:09:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 504933376 bytes 2026-02-20T22:09:28.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 571984896 bytes 2026-02-20T22:09:28.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 628105216 bytes 2026-02-20T22:09:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 662631424 bytes 2026-02-20T22:09:28.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 673979392 bytes 2026-02-20T22:09:28.767 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 745315328 bytes 2026-02-20T22:09:28.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 760919040 bytes 2026-02-20T22:09:28.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 802938880 bytes 2026-02-20T22:09:28.906 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 846815232 bytes 2026-02-20T22:09:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 871097344 bytes 2026-02-20T22:09:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 901861376 bytes 2026-02-20T22:09:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 969696256 bytes 2026-02-20T22:09:29.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1009958912 bytes 2026-02-20T22:09:29.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1081282560 bytes 2026-02-20T22:09:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1123761152 bytes 2026-02-20T22:09:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1130526720 bytes 2026-02-20T22:09:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1156766720 bytes 2026-02-20T22:09:29.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1215545344 bytes 2026-02-20T22:09:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1294080000 bytes 2026-02-20T22:09:29.441 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1311602688 bytes 2026-02-20T22:09:29.501 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1378053120 bytes 2026-02-20T22:09:29.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1389679616 bytes 2026-02-20T22:09:29.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1474081792 bytes 2026-02-20T22:09:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1514294272 bytes 2026-02-20T22:09:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1514924032 bytes 2026-02-20T22:09:29.721 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1540139008 bytes 2026-02-20T22:09:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1560578048 bytes 2026-02-20T22:09:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1639858176 bytes 2026-02-20T22:09:29.864 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1672299520 bytes 2026-02-20T22:09:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1681243136 bytes 2026-02-20T22:09:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1773440000 bytes 2026-02-20T22:09:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1798866944 bytes 2026-02-20T22:09:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1849513984 bytes 2026-02-20T22:09:30.103 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1879335936 bytes 2026-02-20T22:09:30.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1938339840 bytes 2026-02-20T22:09:30.208 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1986390016 bytes 2026-02-20T22:09:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1986973696 bytes 2026-02-20T22:09:30.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2073037824 bytes 2026-02-20T22:09:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2083811328 bytes 2026-02-20T22:09:30.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2112084992 bytes 2026-02-20T22:09:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2171997184 bytes 2026-02-20T22:09:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2194614272 bytes 2026-02-20T22:09:30.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2284967936 bytes 2026-02-20T22:09:30.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2315823104 bytes 2026-02-20T22:09:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2398155776 bytes 2026-02-20T22:09:30.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2447940608 bytes 2026-02-20T22:09:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2461068288 bytes 2026-02-20T22:09:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2498045952 bytes 2026-02-20T22:09:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2505811968 bytes 2026-02-20T22:09:30.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2609372160 bytes 2026-02-20T22:09:30.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2626512896 bytes 2026-02-20T22:09:31.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2697495552 bytes 2026-02-20T22:09:31.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2722783232 bytes 2026-02-20T22:09:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2724310016 bytes 2026-02-20T22:09:31.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2769114112 bytes 2026-02-20T22:09:31.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2773177344 bytes 2026-02-20T22:09:31.232 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2857833472 bytes 2026-02-20T22:09:31.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2902664192 bytes 2026-02-20T22:09:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2914187264 bytes 2026-02-20T22:09:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3026456576 bytes 2026-02-20T22:09:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3076623360 bytes 2026-02-20T22:09:31.498 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3128415232 bytes 2026-02-20T22:09:31.548 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3177663488 bytes 2026-02-20T22:09:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3230416896 bytes 2026-02-20T22:09:31.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3287703552 bytes 2026-02-20T22:09:31.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3316885504 bytes 2026-02-20T22:09:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3347954688 bytes 2026-02-20T22:09:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3422034944 bytes 2026-02-20T22:09:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3453795328 bytes 2026-02-20T22:09:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3523978240 bytes 2026-02-20T22:09:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3542829056 bytes 2026-02-20T22:09:31.983 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3566886912 bytes 2026-02-20T22:09:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3576162304 bytes 2026-02-20T22:09:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3603055616 bytes 2026-02-20T22:09:32.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3653337088 bytes 2026-02-20T22:09:32.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3699963904 bytes 2026-02-20T22:09:32.199 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3706964992 bytes 2026-02-20T22:09:32.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3809570816 bytes 2026-02-20T22:09:32.328 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3867840512 bytes 2026-02-20T22:09:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3912113152 bytes 2026-02-20T22:09:32.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3924923392 bytes 2026-02-20T22:09:32.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3971494912 bytes 2026-02-20T22:09:32.515 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4002837504 bytes 2026-02-20T22:09:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4031133696 bytes 2026-02-20T22:09:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4037390336 bytes 2026-02-20T22:09:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4084070400 bytes 2026-02-20T22:09:32.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4118249472 bytes 2026-02-20T22:09:32.760 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4234270720 bytes 2026-02-20T22:09:32.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4263171072 bytes 2026-02-20T22:09:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4305368064 bytes 2026-02-20T22:09:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4345274368 bytes 2026-02-20T22:09:32.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4430798848 bytes 2026-02-20T22:09:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4454985728 bytes 2026-02-20T22:09:33.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4492443648 bytes 2026-02-20T22:09:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4543355904 bytes 2026-02-20T22:09:33.157 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4595687424 bytes 2026-02-20T22:09:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4665296896 bytes 2026-02-20T22:09:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4721731584 bytes 2026-02-20T22:09:33.315 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4772034560 bytes 2026-02-20T22:09:33.378 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4850776064 bytes 2026-02-20T22:09:33.414 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4863414272 bytes 2026-02-20T22:09:33.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4885109760 bytes 2026-02-20T22:09:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4968337408 bytes 2026-02-20T22:09:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5008954368 bytes 2026-02-20T22:09:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5066394624 bytes 2026-02-20T22:09:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5086553088 bytes 2026-02-20T22:09:33.701 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5107255296 bytes 2026-02-20T22:09:33.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5136508928 bytes 2026-02-20T22:09:33.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5163238400 bytes 2026-02-20T22:09:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5210783744 bytes 2026-02-20T22:09:33.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5259843584 bytes 2026-02-20T22:09:33.930 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5276017664 bytes 2026-02-20T22:09:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5304457216 bytes 2026-02-20T22:09:34.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5342652416 bytes 2026-02-20T22:09:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5404207104 bytes 2026-02-20T22:09:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5404812288 bytes 2026-02-20T22:09:34.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5412574208 bytes 2026-02-20T22:09:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5413165056 bytes 2026-02-20T22:09:34.229 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5482417152 bytes 2026-02-20T22:09:34.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5608353792 bytes 2026-02-20T22:09:34.341 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5611720704 bytes 2026-02-20T22:09:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5618727936 bytes 2026-02-20T22:09:34.420 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5646921728 bytes 2026-02-20T22:09:34.465 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5676002304 bytes 2026-02-20T22:09:34.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5693438976 bytes 2026-02-20T22:09:34.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5694763008 bytes 2026-02-20T22:09:34.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5711293440 bytes 2026-02-20T22:09:34.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5765520384 bytes 2026-02-20T22:09:34.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5811727360 bytes 2026-02-20T22:09:34.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5870220288 bytes 2026-02-20T22:09:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5883308032 bytes 2026-02-20T22:09:34.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5974170624 bytes 2026-02-20T22:09:34.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6033293312 bytes 2026-02-20T22:09:34.924 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6051248128 bytes 2026-02-20T22:09:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6117592064 bytes 2026-02-20T22:09:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6201181184 bytes 2026-02-20T22:09:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6256808960 bytes 2026-02-20T22:09:35.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6298188800 bytes 2026-02-20T22:09:35.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6399144960 bytes 2026-02-20T22:09:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6423336960 bytes 2026-02-20T22:09:35.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6433719296 bytes 2026-02-20T22:09:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6456879104 bytes 2026-02-20T22:09:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6498380800 bytes 2026-02-20T22:09:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6538317824 bytes 2026-02-20T22:09:35.502 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6622132224 bytes 2026-02-20T22:09:35.550 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6656634880 bytes 2026-02-20T22:09:35.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6722208768 bytes 2026-02-20T22:09:35.647 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6744025088 bytes 2026-02-20T22:09:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6827696128 bytes 2026-02-20T22:09:35.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6860454912 bytes 2026-02-20T22:09:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6913808384 bytes 2026-02-20T22:09:35.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6953708544 bytes 2026-02-20T22:09:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7004878848 bytes 2026-02-20T22:09:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7015256064 bytes 2026-02-20T22:09:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7043921920 bytes 2026-02-20T22:09:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7113876480 bytes 2026-02-20T22:09:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7150903296 bytes 2026-02-20T22:09:36.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7216317440 bytes 2026-02-20T22:09:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7236167680 bytes 2026-02-20T22:09:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7305206784 bytes 2026-02-20T22:09:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7426447360 bytes 2026-02-20T22:09:36.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7519860736 bytes 2026-02-20T22:09:36.435 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7543384064 bytes 2026-02-20T22:09:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7566351360 bytes 2026-02-20T22:09:36.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7644983296 bytes 2026-02-20T22:09:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7724025856 bytes 2026-02-20T22:09:36.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7846051840 bytes 2026-02-20T22:09:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7875348480 bytes 2026-02-20T22:09:36.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7948178432 bytes 2026-02-20T22:09:36.821 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7965294592 bytes 2026-02-20T22:09:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 8036726784 bytes 2026-02-20T22:09:36.921 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 8050159616 bytes 2026-02-20T22:09:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 8075501568 bytes 2026-02-20T22:09:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 8112931840 bytes 2026-02-20T22:09:37.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 8154423296 bytes 2026-02-20T22:09:37.096 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8169388032 bytes 2026-02-20T22:09:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8192120832 bytes 2026-02-20T22:09:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8212910080 bytes 2026-02-20T22:09:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8221461504 bytes 2026-02-20T22:09:37.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8236832768 bytes 2026-02-20T22:09:37.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8326479872 bytes 2026-02-20T22:09:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8340170752 bytes 2026-02-20T22:09:37.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8379063296 bytes 2026-02-20T22:09:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8411571200 bytes 2026-02-20T22:09:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8519030784 bytes 2026-02-20T22:09:37.562 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8542025728 bytes 2026-02-20T22:09:37.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8574940160 bytes 2026-02-20T22:09:37.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8646266880 bytes 2026-02-20T22:09:37.724 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8728030208 bytes 2026-02-20T22:09:37.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8792786944 bytes 2026-02-20T22:09:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8905570304 bytes 2026-02-20T22:09:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8962651136 bytes 2026-02-20T22:09:37.962 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 9002621952 bytes 2026-02-20T22:09:38.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 9090839552 bytes 2026-02-20T22:09:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 9157202944 bytes 2026-02-20T22:09:38.132 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 9183978496 bytes 2026-02-20T22:09:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 9225500672 bytes 2026-02-20T22:09:38.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9234704384 bytes 2026-02-20T22:09:38.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9273566208 bytes 2026-02-20T22:09:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9316189184 bytes 2026-02-20T22:09:38.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9332744192 bytes 2026-02-20T22:09:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9356028928 bytes 2026-02-20T22:09:38.445 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9399081984 bytes 2026-02-20T22:09:38.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9400364032 bytes 2026-02-20T22:09:38.537 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9436462080 bytes 2026-02-20T22:09:38.598 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9504353280 bytes 2026-02-20T22:09:38.661 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9583810560 bytes 2026-02-20T22:09:38.734 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9690028032 bytes 2026-02-20T22:09:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9703538688 bytes 2026-02-20T22:09:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9723264000 bytes 2026-02-20T22:09:38.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:09:38.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:09:38.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:09:38.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:09:38.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_6 2026-02-20T22:09:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T22:09:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:09:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b0863e1bbc 2026-02-20T22:09:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:09:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:09:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_6 present 10b0863e1bbc 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b0863e1bbc 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:09:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:09:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_6 2026-02-20T22:09:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T22:09:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:09:38.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b0863e1bbc 2026-02-20T22:09:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b0863e1bbc 2026-02-20T22:09:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b0863e1bbc -o 10b0863e1bbc = 10b0863e1bbc 2026-02-20T22:09:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:09:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:09:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:09:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_6 2026-02-20T22:09:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:09:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:09:38.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:09:38.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2026-02-20T22:09:41.237 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4102154818509106) > weight (0.25) 2026-02-20T22:09:41.238 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3167977662011775) > weight (0.25) 2026-02-20T22:09:41.238 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.2981854460345298) > weight (0.25) 2026-02-20T22:09:41.238 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.4482597975219532) > weight (0.25) 2026-02-20T22:09:41.238 INFO:tasks.rbd_mirror_thrash:waiting for 74.45402376813304 secs before thrashing 2026-02-20T22:09:54.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:09:54.093+0000 7f147ea66640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55f0ee61fa00 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:09:54.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:09:54.093+0000 7f1485273640 -1 rbd::mirror::ImageReplayer: 0x55f0edf7d900 [3/51b51193-d420-435d-bd9f-2a836d56de53] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:10:04.078 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9162786090579733) > weight (0.25) 2026-02-20T22:10:04.079 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.262619607568341) > weight (0.25) 2026-02-20T22:10:04.079 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-20T22:10:04.079 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-20T22:10:04.079 INFO:tasks.rbd_mirror_thrash:waiting for 6.810010700790703 secs before reviving daemons 2026-02-20T22:10:04.080 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:10:04.077+0000 7f1485a74640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 32259) UID: 1000 2026-02-20T22:10:08.114 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:10:08.112+0000 7fc7a2858640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55e036d57040 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:10:08.114 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:10:08.112+0000 7fc7a9065640 -1 rbd::mirror::ImageReplayer: 0x55e0373a1b80 [3/51b51193-d420-435d-bd9f-2a836d56de53] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:10:09.767 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:10:09.764+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b292800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=190 2026-02-20T22:10:09.767 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:10:09.764+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b292800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=193, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=be, last_copied_object_number=128, snap_seqs={190=18446744073709551614}] 2026-02-20T22:10:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:10:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-20T22:10:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:10:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:10:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:10:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:10:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:10:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:10:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T22:10:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:10:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:10:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_6 2026-02-20T22:10:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T22:10:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:10:09.807 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-20T22:10:09.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:10:09.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:10:09.810 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:10:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-20T22:10:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:10:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:09.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_6 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T22:10:09.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:10:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-20T22:10:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:10:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:10:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:10:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:10:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_6 2026-02-20T22:10:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:10:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2026-02-20T22:10:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-20T22:10:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:10:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:10:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:10:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_6 2026-02-20T22:10:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_6 2026-02-20T22:10:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-20T22:10:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-20T22:10:10.160 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:10:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:10:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:10:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_7 2026-02-20T22:10:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_7 512M 2026-02-20T22:10:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:10:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:10:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:10:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:10:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:10:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:10:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:10:10.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_7 2026-02-20T22:10:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_7 2026-02-20T22:10:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:10:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 snapshot 2026-02-20T22:10:10.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:10:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-02-20T22:10:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_7': 2026-02-20T22:10:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:10:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:10:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:10:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 199188abfce6 2026-02-20T22:10:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.199188abfce6 2026-02-20T22:10:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:10:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:10:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:10:10.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:10:10.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:10:10 2026 2026-02-20T22:10:10.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:10:10 2026 2026-02-20T22:10:10.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:10:10 2026 2026-02-20T22:10:10.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:10:10.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:10:10.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 0b5c4d6a-8951-40e8-96ab-513b4e7c03de 2026-02-20T22:10:10.825 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:10:10.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-20T22:10:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:10.827 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:10:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=8 2026-02-20T22:10:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:10:10.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:10:10.890 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-20T22:10:10.890 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T22:10:10.891 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-20T22:10:10.891 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T22:10:10.891 DEBUG:teuthology.orchestra.run.trial107:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T22:10:10.893 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T22:10:10.893 INFO:tasks.rbd_mirror_thrash:waiting for 104.65701383940686 secs before thrashing 2026-02-20T22:10:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 48678912 bytes 2026-02-20T22:10:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 78579712 bytes 2026-02-20T22:10:11.067 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 110947328 bytes 2026-02-20T22:10:11.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 141836288 bytes 2026-02-20T22:10:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 176913408 bytes 2026-02-20T22:10:11.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 244914176 bytes 2026-02-20T22:10:11.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 271920128 bytes 2026-02-20T22:10:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 292696064 bytes 2026-02-20T22:10:11.339 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 341211136 bytes 2026-02-20T22:10:11.397 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 411651072 bytes 2026-02-20T22:10:11.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 476892160 bytes 2026-02-20T22:10:11.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 514746368 bytes 2026-02-20T22:10:11.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 538797056 bytes 2026-02-20T22:10:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 599793664 bytes 2026-02-20T22:10:11.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 659632128 bytes 2026-02-20T22:10:11.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 737004544 bytes 2026-02-20T22:10:11.748 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 778497024 bytes 2026-02-20T22:10:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 813876224 bytes 2026-02-20T22:10:11.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 838237184 bytes 2026-02-20T22:10:11.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 893373440 bytes 2026-02-20T22:10:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 927930368 bytes 2026-02-20T22:10:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 968404992 bytes 2026-02-20T22:10:12.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 970844160 bytes 2026-02-20T22:10:12.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 979595264 bytes 2026-02-20T22:10:12.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1004510208 bytes 2026-02-20T22:10:12.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1067825152 bytes 2026-02-20T22:10:12.203 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1124530176 bytes 2026-02-20T22:10:12.243 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1161364480 bytes 2026-02-20T22:10:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1224356864 bytes 2026-02-20T22:10:12.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1264325632 bytes 2026-02-20T22:10:12.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1327007744 bytes 2026-02-20T22:10:12.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1327621120 bytes 2026-02-20T22:10:12.487 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1375814656 bytes 2026-02-20T22:10:12.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1403333632 bytes 2026-02-20T22:10:12.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1426797568 bytes 2026-02-20T22:10:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1452945408 bytes 2026-02-20T22:10:12.676 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1527775232 bytes 2026-02-20T22:10:12.719 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1554519040 bytes 2026-02-20T22:10:12.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1577918464 bytes 2026-02-20T22:10:12.793 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1586772992 bytes 2026-02-20T22:10:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1664224256 bytes 2026-02-20T22:10:12.901 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1711332352 bytes 2026-02-20T22:10:12.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1768715264 bytes 2026-02-20T22:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1791630336 bytes 2026-02-20T22:10:13.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1809332224 bytes 2026-02-20T22:10:13.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1826735104 bytes 2026-02-20T22:10:13.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1896408064 bytes 2026-02-20T22:10:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1942596608 bytes 2026-02-20T22:10:13.224 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1964948480 bytes 2026-02-20T22:10:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 1975646208 bytes 2026-02-20T22:10:13.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 1991297024 bytes 2026-02-20T22:10:13.350 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2051634176 bytes 2026-02-20T22:10:13.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2085712896 bytes 2026-02-20T22:10:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2152931328 bytes 2026-02-20T22:10:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2181136384 bytes 2026-02-20T22:10:13.557 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2262369280 bytes 2026-02-20T22:10:13.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2311036928 bytes 2026-02-20T22:10:13.657 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2354518016 bytes 2026-02-20T22:10:13.706 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2399802368 bytes 2026-02-20T22:10:13.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2453975040 bytes 2026-02-20T22:10:13.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2515860480 bytes 2026-02-20T22:10:13.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2572635136 bytes 2026-02-20T22:10:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2634715136 bytes 2026-02-20T22:10:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2672982016 bytes 2026-02-20T22:10:14.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2709293056 bytes 2026-02-20T22:10:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2740524032 bytes 2026-02-20T22:10:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2780583936 bytes 2026-02-20T22:10:14.354 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2804469760 bytes 2026-02-20T22:10:14.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2842060800 bytes 2026-02-20T22:10:14.466 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2919767040 bytes 2026-02-20T22:10:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2972096512 bytes 2026-02-20T22:10:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3011773440 bytes 2026-02-20T22:10:14.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3012390912 bytes 2026-02-20T22:10:14.656 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3058809856 bytes 2026-02-20T22:10:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3112557568 bytes 2026-02-20T22:10:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3176748032 bytes 2026-02-20T22:10:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3195894784 bytes 2026-02-20T22:10:14.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3278461952 bytes 2026-02-20T22:10:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3344437248 bytes 2026-02-20T22:10:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3404871680 bytes 2026-02-20T22:10:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3424974848 bytes 2026-02-20T22:10:15.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3468411904 bytes 2026-02-20T22:10:15.134 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3521353728 bytes 2026-02-20T22:10:15.185 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3576071168 bytes 2026-02-20T22:10:15.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3620154368 bytes 2026-02-20T22:10:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3674727424 bytes 2026-02-20T22:10:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3758331904 bytes 2026-02-20T22:10:15.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3804162048 bytes 2026-02-20T22:10:15.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3849483264 bytes 2026-02-20T22:10:15.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3888284672 bytes 2026-02-20T22:10:15.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3899754496 bytes 2026-02-20T22:10:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3971794944 bytes 2026-02-20T22:10:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4018847744 bytes 2026-02-20T22:10:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4123599872 bytes 2026-02-20T22:10:15.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4146570240 bytes 2026-02-20T22:10:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4250737664 bytes 2026-02-20T22:10:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4278539264 bytes 2026-02-20T22:10:15.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4305474560 bytes 2026-02-20T22:10:15.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4317198336 bytes 2026-02-20T22:10:15.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4347313152 bytes 2026-02-20T22:10:16.018 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4348841984 bytes 2026-02-20T22:10:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4355324928 bytes 2026-02-20T22:10:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4442569728 bytes 2026-02-20T22:10:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4470305792 bytes 2026-02-20T22:10:16.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4526019584 bytes 2026-02-20T22:10:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4565319680 bytes 2026-02-20T22:10:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4666976256 bytes 2026-02-20T22:10:16.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4704114688 bytes 2026-02-20T22:10:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4733918208 bytes 2026-02-20T22:10:16.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4816166912 bytes 2026-02-20T22:10:16.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4849153024 bytes 2026-02-20T22:10:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4914718720 bytes 2026-02-20T22:10:16.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4935696384 bytes 2026-02-20T22:10:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4978712576 bytes 2026-02-20T22:10:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5004939264 bytes 2026-02-20T22:10:16.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5038370816 bytes 2026-02-20T22:10:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5096020992 bytes 2026-02-20T22:10:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5174201344 bytes 2026-02-20T22:10:16.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5232103424 bytes 2026-02-20T22:10:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5232728064 bytes 2026-02-20T22:10:17.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5252085760 bytes 2026-02-20T22:10:17.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5297961984 bytes 2026-02-20T22:10:17.125 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5367530496 bytes 2026-02-20T22:10:17.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5418247168 bytes 2026-02-20T22:10:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5444267008 bytes 2026-02-20T22:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5459969024 bytes 2026-02-20T22:10:17.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5510832128 bytes 2026-02-20T22:10:17.359 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5555918848 bytes 2026-02-20T22:10:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5565062144 bytes 2026-02-20T22:10:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5589599232 bytes 2026-02-20T22:10:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5622112256 bytes 2026-02-20T22:10:17.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5704892416 bytes 2026-02-20T22:10:17.592 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5744563200 bytes 2026-02-20T22:10:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5755165696 bytes 2026-02-20T22:10:17.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5798111232 bytes 2026-02-20T22:10:17.764 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5925080064 bytes 2026-02-20T22:10:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5994750976 bytes 2026-02-20T22:10:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 6025309184 bytes 2026-02-20T22:10:17.902 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 6037898240 bytes 2026-02-20T22:10:17.941 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6061712384 bytes 2026-02-20T22:10:18.003 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6125481984 bytes 2026-02-20T22:10:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6186210304 bytes 2026-02-20T22:10:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6269359104 bytes 2026-02-20T22:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6295050240 bytes 2026-02-20T22:10:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6385037312 bytes 2026-02-20T22:10:18.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6427486208 bytes 2026-02-20T22:10:18.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6461009920 bytes 2026-02-20T22:10:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6492951552 bytes 2026-02-20T22:10:18.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6524466176 bytes 2026-02-20T22:10:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6610877440 bytes 2026-02-20T22:10:18.523 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6650514432 bytes 2026-02-20T22:10:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6696066048 bytes 2026-02-20T22:10:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6765314048 bytes 2026-02-20T22:10:18.667 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6777544704 bytes 2026-02-20T22:10:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6778160128 bytes 2026-02-20T22:10:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6828532736 bytes 2026-02-20T22:10:18.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6850111488 bytes 2026-02-20T22:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T22:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_7 2026-02-20T22:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T22:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:10:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b0e3cde217 2026-02-20T22:10:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:10:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:10:18.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_7 present 10b0e3cde217 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b0e3cde217 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:10:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:10:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_7 2026-02-20T22:10:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T22:10:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:10:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b0e3cde217 2026-02-20T22:10:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b0e3cde217 2026-02-20T22:10:18.996 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b0e3cde217 -o 10b0e3cde217 = 10b0e3cde217 2026-02-20T22:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_7 2026-02-20T22:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:18.997 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2026-02-20T22:10:46.116 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:10:46.112+0000 7fc7a2057640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55e036d57520 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:10:46.116 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:10:46.112+0000 7fc7a9065640 -1 rbd::mirror::ImageReplayer: 0x55e036e84000 [3/0b5c4d6a-8951-40e8-96ab-513b4e7c03de] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:10:49.799 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:10:49.796+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b1dd400 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=191 2026-02-20T22:10:49.799 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:10:49.796+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7b1dd400 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=194, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=bf, last_copied_object_number=128, snap_seqs={191=18446744073709551614}] 2026-02-20T22:10:49.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:10:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-20T22:10:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:10:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:10:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:10:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:10:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:10:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T22:10:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:10:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:10:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_7 2026-02-20T22:10:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T22:10:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:10:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-20T22:10:49.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:10:49.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:10:49.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:10:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-20T22:10:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:10:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:10:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:10:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:10:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:10:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T22:10:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:10:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:10:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_7 2026-02-20T22:10:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T22:10:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:10:49.965 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-20T22:10:49.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:10:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:10:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:10:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:10:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_7 2026-02-20T22:10:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2026-02-20T22:10:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-20T22:10:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:10:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:10:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_7 2026-02-20T22:10:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_7 2026-02-20T22:10:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-20T22:10:50.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-20T22:10:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:10:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:10:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:10:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_8 2026-02-20T22:10:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_8 512M 2026-02-20T22:10:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:10:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:10:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:10:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:10:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:10:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:10:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:10:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:10:50.189 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_8 2026-02-20T22:10:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_8 2026-02-20T22:10:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:10:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:10:50.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 snapshot 2026-02-20T22:10:50.815 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:10:50.824 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-02-20T22:10:50.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_8': 2026-02-20T22:10:50.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:10:50.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:10:50.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:10:50.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 19b3a65bb32 2026-02-20T22:10:50.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.19b3a65bb32 2026-02-20T22:10:50.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:10:50.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:10:50.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:10:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:10:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:10:50 2026 2026-02-20T22:10:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:10:50 2026 2026-02-20T22:10:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:10:50 2026 2026-02-20T22:10:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:10:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:10:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 0aed9a68-b18e-4d7d-a088-4396217ccce9 2026-02-20T22:10:50.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:10:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-20T22:10:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:10:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:10:50.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:10:50.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=9 2026-02-20T22:10:50.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:10:50.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:10:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 29585408 bytes 2026-02-20T22:10:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 57268224 bytes 2026-02-20T22:10:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 67082240 bytes 2026-02-20T22:10:51.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 79007744 bytes 2026-02-20T22:10:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 113678336 bytes 2026-02-20T22:10:51.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 133762048 bytes 2026-02-20T22:10:51.252 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 196894720 bytes 2026-02-20T22:10:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 202859520 bytes 2026-02-20T22:10:51.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 255873024 bytes 2026-02-20T22:10:51.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 280158208 bytes 2026-02-20T22:10:51.417 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 306079744 bytes 2026-02-20T22:10:51.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 354055168 bytes 2026-02-20T22:10:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 458727424 bytes 2026-02-20T22:10:51.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 493965312 bytes 2026-02-20T22:10:51.636 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 554711040 bytes 2026-02-20T22:10:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 619181056 bytes 2026-02-20T22:10:51.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 655542272 bytes 2026-02-20T22:10:51.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 696569856 bytes 2026-02-20T22:10:51.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 764235776 bytes 2026-02-20T22:10:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 818612224 bytes 2026-02-20T22:10:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 889042944 bytes 2026-02-20T22:10:51.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 906830848 bytes 2026-02-20T22:10:52.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 912210944 bytes 2026-02-20T22:10:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 951945216 bytes 2026-02-20T22:10:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 999288832 bytes 2026-02-20T22:10:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1032087552 bytes 2026-02-20T22:10:52.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1055023104 bytes 2026-02-20T22:10:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1102683136 bytes 2026-02-20T22:10:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1158653952 bytes 2026-02-20T22:10:52.359 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1177337856 bytes 2026-02-20T22:10:52.403 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1226975232 bytes 2026-02-20T22:10:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1237385216 bytes 2026-02-20T22:10:52.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1246193664 bytes 2026-02-20T22:10:52.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1315612672 bytes 2026-02-20T22:10:52.584 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1375660032 bytes 2026-02-20T22:10:52.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1455532032 bytes 2026-02-20T22:10:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1479134208 bytes 2026-02-20T22:10:52.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1520980992 bytes 2026-02-20T22:10:52.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1598110720 bytes 2026-02-20T22:10:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1653743616 bytes 2026-02-20T22:10:52.897 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1671958528 bytes 2026-02-20T22:10:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1720341504 bytes 2026-02-20T22:10:53.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1799284736 bytes 2026-02-20T22:10:53.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1845466112 bytes 2026-02-20T22:10:53.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1888706560 bytes 2026-02-20T22:10:53.162 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1944297472 bytes 2026-02-20T22:10:53.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1954757632 bytes 2026-02-20T22:10:53.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1969665024 bytes 2026-02-20T22:10:53.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1990985728 bytes 2026-02-20T22:10:53.340 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2072201216 bytes 2026-02-20T22:10:53.346 INFO:tasks.ceph.cluster2.osd.0.trial107.stderr:2026-02-20T22:10:53.344+0000 7fa5e2658640 -1 reset not still connected to 0x55b9295e0750 2026-02-20T22:10:53.380 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2093244416 bytes 2026-02-20T22:10:53.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2106346496 bytes 2026-02-20T22:10:53.455 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2125650944 bytes 2026-02-20T22:10:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2179196928 bytes 2026-02-20T22:10:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2189849600 bytes 2026-02-20T22:10:53.581 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2212497408 bytes 2026-02-20T22:10:53.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2300366848 bytes 2026-02-20T22:10:53.687 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2323505152 bytes 2026-02-20T22:10:53.744 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2374587392 bytes 2026-02-20T22:10:53.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2426801152 bytes 2026-02-20T22:10:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2513185792 bytes 2026-02-20T22:10:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2556694528 bytes 2026-02-20T22:10:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2620727296 bytes 2026-02-20T22:10:54.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2663001088 bytes 2026-02-20T22:10:54.051 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2674507776 bytes 2026-02-20T22:10:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2731947008 bytes 2026-02-20T22:10:54.153 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2774332416 bytes 2026-02-20T22:10:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2808541184 bytes 2026-02-20T22:10:54.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2826600448 bytes 2026-02-20T22:10:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2931155968 bytes 2026-02-20T22:10:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2959581184 bytes 2026-02-20T22:10:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2987240448 bytes 2026-02-20T22:10:54.431 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3010160640 bytes 2026-02-20T22:10:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3013378048 bytes 2026-02-20T22:10:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3067198464 bytes 2026-02-20T22:10:54.570 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3129811968 bytes 2026-02-20T22:10:54.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3159660544 bytes 2026-02-20T22:10:54.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3231600640 bytes 2026-02-20T22:10:54.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3298706432 bytes 2026-02-20T22:10:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3328206848 bytes 2026-02-20T22:10:54.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3384312832 bytes 2026-02-20T22:10:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3390205952 bytes 2026-02-20T22:10:54.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3410665472 bytes 2026-02-20T22:10:54.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3431589888 bytes 2026-02-20T22:10:54.994 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3470787584 bytes 2026-02-20T22:10:55.049 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3532023808 bytes 2026-02-20T22:10:55.094 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3569409024 bytes 2026-02-20T22:10:55.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3691140096 bytes 2026-02-20T22:10:55.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3716160512 bytes 2026-02-20T22:10:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3733531648 bytes 2026-02-20T22:10:55.296 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3752460288 bytes 2026-02-20T22:10:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3836773376 bytes 2026-02-20T22:10:55.410 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3891447808 bytes 2026-02-20T22:10:55.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3912701952 bytes 2026-02-20T22:10:55.491 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3935932416 bytes 2026-02-20T22:10:55.693 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.457057771447828) > weight (0.25) 2026-02-20T22:10:55.693 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.3042198372821716) > weight (0.25) 2026-02-20T22:10:55.693 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6050059160278606) > weight (0.25) 2026-02-20T22:10:55.693 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.8999673743483124) > weight (0.25) 2026-02-20T22:10:55.693 INFO:tasks.rbd_mirror_thrash:waiting for 67.572046717203 secs before thrashing 2026-02-20T22:10:55.746 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 3981733888 bytes 2026-02-20T22:10:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4015318016 bytes 2026-02-20T22:10:55.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4066766848 bytes 2026-02-20T22:10:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4120249344 bytes 2026-02-20T22:10:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4130440192 bytes 2026-02-20T22:10:55.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4163431424 bytes 2026-02-20T22:10:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4240658432 bytes 2026-02-20T22:10:56.100 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4277319680 bytes 2026-02-20T22:10:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4301604864 bytes 2026-02-20T22:10:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4350889984 bytes 2026-02-20T22:10:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4396632064 bytes 2026-02-20T22:10:56.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4434388992 bytes 2026-02-20T22:10:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4493329408 bytes 2026-02-20T22:10:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4494273536 bytes 2026-02-20T22:10:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4527630336 bytes 2026-02-20T22:10:56.452 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4545034240 bytes 2026-02-20T22:10:56.509 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4613132288 bytes 2026-02-20T22:10:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4655085568 bytes 2026-02-20T22:10:56.608 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4695872512 bytes 2026-02-20T22:10:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4709826560 bytes 2026-02-20T22:10:56.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4746553344 bytes 2026-02-20T22:10:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4789617664 bytes 2026-02-20T22:10:56.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4876623872 bytes 2026-02-20T22:10:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4965380096 bytes 2026-02-20T22:10:56.907 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4965998592 bytes 2026-02-20T22:10:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4995872768 bytes 2026-02-20T22:10:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5006990336 bytes 2026-02-20T22:10:57.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5032663040 bytes 2026-02-20T22:10:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5061012480 bytes 2026-02-20T22:10:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5063107584 bytes 2026-02-20T22:10:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5138209792 bytes 2026-02-20T22:10:57.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5186491392 bytes 2026-02-20T22:10:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5221011456 bytes 2026-02-20T22:10:57.301 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5221625856 bytes 2026-02-20T22:10:57.362 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5304326144 bytes 2026-02-20T22:10:57.415 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5348335616 bytes 2026-02-20T22:10:57.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5371372544 bytes 2026-02-20T22:10:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5420652544 bytes 2026-02-20T22:10:57.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5456561152 bytes 2026-02-20T22:10:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5481694208 bytes 2026-02-20T22:10:57.668 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5575229440 bytes 2026-02-20T22:10:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5576178688 bytes 2026-02-20T22:10:57.759 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5639079936 bytes 2026-02-20T22:10:57.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5672927232 bytes 2026-02-20T22:10:57.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5726826496 bytes 2026-02-20T22:10:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5748071424 bytes 2026-02-20T22:10:57.940 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5802851328 bytes 2026-02-20T22:10:57.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5827636224 bytes 2026-02-20T22:10:58.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5851310080 bytes 2026-02-20T22:10:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5978924032 bytes 2026-02-20T22:10:58.151 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5997393920 bytes 2026-02-20T22:10:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6042900480 bytes 2026-02-20T22:10:58.245 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6069476352 bytes 2026-02-20T22:10:58.287 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6094604288 bytes 2026-02-20T22:10:58.336 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6144848896 bytes 2026-02-20T22:10:58.387 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6202269696 bytes 2026-02-20T22:10:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6267415552 bytes 2026-02-20T22:10:58.499 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6308283392 bytes 2026-02-20T22:10:58.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6414330880 bytes 2026-02-20T22:10:58.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6492909568 bytes 2026-02-20T22:10:58.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6556057600 bytes 2026-02-20T22:10:58.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6595465216 bytes 2026-02-20T22:10:58.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6652850176 bytes 2026-02-20T22:10:58.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6682759168 bytes 2026-02-20T22:10:58.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6737012736 bytes 2026-02-20T22:10:58.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6750108672 bytes 2026-02-20T22:10:58.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6766383104 bytes 2026-02-20T22:10:58.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6772251648 bytes 2026-02-20T22:10:59.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6863469568 bytes 2026-02-20T22:10:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6885552128 bytes 2026-02-20T22:10:59.164 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6956332032 bytes 2026-02-20T22:10:59.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7014251520 bytes 2026-02-20T22:10:59.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7088458752 bytes 2026-02-20T22:10:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7144932352 bytes 2026-02-20T22:10:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7173918720 bytes 2026-02-20T22:10:59.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7232115712 bytes 2026-02-20T22:10:59.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7232746496 bytes 2026-02-20T22:10:59.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7316784128 bytes 2026-02-20T22:10:59.589 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7396096000 bytes 2026-02-20T22:10:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7441759232 bytes 2026-02-20T22:10:59.677 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7456884736 bytes 2026-02-20T22:10:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7547117568 bytes 2026-02-20T22:10:59.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7568559104 bytes 2026-02-20T22:10:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7594067968 bytes 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:10:59.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:10:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:10:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:10:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T22:10:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:10:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:10:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_8 2026-02-20T22:10:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T22:10:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:10:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b04d5ef50c 2026-02-20T22:10:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:10:59.889 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:10:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_8 present 10b04d5ef50c 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b04d5ef50c 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:10:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:10:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_8 2026-02-20T22:10:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T22:10:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:11:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b04d5ef50c 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b04d5ef50c 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b04d5ef50c -o 10b04d5ef50c = 10b04d5ef50c 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_8 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:11:00.021 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2026-02-20T22:11:16.115 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:11:16.111+0000 7fc7a2858640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55e036bc9d40 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:11:16.115 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:11:16.111+0000 7fc7a8864640 -1 rbd::mirror::ImageReplayer: 0x55e0373a1b80 [3/0aed9a68-b18e-4d7d-a088-4396217ccce9] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:11:30.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:11:30.823+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7d9c9800 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=192 2026-02-20T22:11:30.829 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:11:30.823+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7d9c9800 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=195, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=c0, last_copied_object_number=128, snap_seqs={192=18446744073709551614}] 2026-02-20T22:11:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:11:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:11:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:11:30.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_8 2026-02-20T22:11:30.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T22:11:30.848 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:11:30.872 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-20T22:11:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:11:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:11:30.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_8 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T22:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:11:30.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-20T22:11:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:11:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:11:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:11:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:11:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_8 2026-02-20T22:11:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:11:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:11:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2026-02-20T22:11:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-20T22:11:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:11:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:11:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:11:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:11:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_8 2026-02-20T22:11:31.033 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:11:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_8 2026-02-20T22:11:31.034 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-20T22:11:31.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-20T22:11:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:11:31.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:11:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:11:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_9 2026-02-20T22:11:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_9 512M 2026-02-20T22:11:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:11:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:11:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:31.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:11:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:11:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:11:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:11:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:11:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:11:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:11:31.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_9 2026-02-20T22:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_9 2026-02-20T22:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 snapshot 2026-02-20T22:11:31.850 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:11:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-02-20T22:11:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_9': 2026-02-20T22:11:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:11:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:11:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:11:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 19cbf8f6060e 2026-02-20T22:11:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.19cbf8f6060e 2026-02-20T22:11:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:11:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:11:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:11:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:11:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:11:31 2026 2026-02-20T22:11:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:11:31 2026 2026-02-20T22:11:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:11:31 2026 2026-02-20T22:11:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:11:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:11:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 2ee8fc72-8ddb-4980-a83f-4c17100a038c 2026-02-20T22:11:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:11:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-20T22:11:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:11:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:31.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:11:31.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:11:31.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=12 2026-02-20T22:11:31.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:11:31.888 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:11:32.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 7027712 bytes 2026-02-20T22:11:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 47457280 bytes 2026-02-20T22:11:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 80532480 bytes 2026-02-20T22:11:32.179 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 119819264 bytes 2026-02-20T22:11:32.238 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 189944832 bytes 2026-02-20T22:11:32.306 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 292368384 bytes 2026-02-20T22:11:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 308143104 bytes 2026-02-20T22:11:32.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 403606528 bytes 2026-02-20T22:11:32.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 419571712 bytes 2026-02-20T22:11:32.508 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 493131776 bytes 2026-02-20T22:11:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 545261568 bytes 2026-02-20T22:11:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 552174592 bytes 2026-02-20T22:11:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 622717952 bytes 2026-02-20T22:11:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 648669184 bytes 2026-02-20T22:11:32.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 715533312 bytes 2026-02-20T22:11:32.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 779805696 bytes 2026-02-20T22:11:32.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 813364224 bytes 2026-02-20T22:11:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 873660416 bytes 2026-02-20T22:11:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 948909056 bytes 2026-02-20T22:11:33.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 983707648 bytes 2026-02-20T22:11:33.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 1023969280 bytes 2026-02-20T22:11:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 1090820096 bytes 2026-02-20T22:11:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 1138732032 bytes 2026-02-20T22:11:33.210 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 1168897024 bytes 2026-02-20T22:11:33.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 1189836800 bytes 2026-02-20T22:11:33.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 1283447808 bytes 2026-02-20T22:11:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1292302336 bytes 2026-02-20T22:11:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1313116160 bytes 2026-02-20T22:11:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1330027520 bytes 2026-02-20T22:11:33.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1334761472 bytes 2026-02-20T22:11:33.525 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1418423296 bytes 2026-02-20T22:11:33.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1461769216 bytes 2026-02-20T22:11:33.622 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1496784896 bytes 2026-02-20T22:11:33.670 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1532553216 bytes 2026-02-20T22:11:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1565581312 bytes 2026-02-20T22:11:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1593439232 bytes 2026-02-20T22:11:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1664336896 bytes 2026-02-20T22:11:33.869 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1690183680 bytes 2026-02-20T22:11:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1703770112 bytes 2026-02-20T22:11:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1773510656 bytes 2026-02-20T22:11:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1783698432 bytes 2026-02-20T22:11:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1839040512 bytes 2026-02-20T22:11:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1887883264 bytes 2026-02-20T22:11:34.158 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1919182848 bytes 2026-02-20T22:11:34.215 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1976598528 bytes 2026-02-20T22:11:34.282 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 2056278016 bytes 2026-02-20T22:11:34.317 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 2063100928 bytes 2026-02-20T22:11:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 2081923072 bytes 2026-02-20T22:11:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 2099300352 bytes 2026-02-20T22:11:34.427 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2113377280 bytes 2026-02-20T22:11:34.474 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2158199808 bytes 2026-02-20T22:11:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2231432192 bytes 2026-02-20T22:11:34.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2274357248 bytes 2026-02-20T22:11:34.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2305228800 bytes 2026-02-20T22:11:34.683 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2373304320 bytes 2026-02-20T22:11:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2393059328 bytes 2026-02-20T22:11:34.766 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2428896256 bytes 2026-02-20T22:11:34.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2495484928 bytes 2026-02-20T22:11:34.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2561513472 bytes 2026-02-20T22:11:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2631967744 bytes 2026-02-20T22:11:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2686491648 bytes 2026-02-20T22:11:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2774759424 bytes 2026-02-20T22:11:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2840205312 bytes 2026-02-20T22:11:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2895791104 bytes 2026-02-20T22:11:35.223 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2924561408 bytes 2026-02-20T22:11:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2958325760 bytes 2026-02-20T22:11:35.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 3012780032 bytes 2026-02-20T22:11:35.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 3021763584 bytes 2026-02-20T22:11:35.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 3160633344 bytes 2026-02-20T22:11:35.489 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 3203914752 bytes 2026-02-20T22:11:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 3286085632 bytes 2026-02-20T22:11:35.593 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 3308576768 bytes 2026-02-20T22:11:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3342126080 bytes 2026-02-20T22:11:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3420184576 bytes 2026-02-20T22:11:35.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3499108352 bytes 2026-02-20T22:11:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3531496448 bytes 2026-02-20T22:11:35.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3577032704 bytes 2026-02-20T22:11:35.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3612094464 bytes 2026-02-20T22:11:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3636652032 bytes 2026-02-20T22:11:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3706932224 bytes 2026-02-20T22:11:36.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3777261568 bytes 2026-02-20T22:11:36.112 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3813937152 bytes 2026-02-20T22:11:36.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3894257664 bytes 2026-02-20T22:11:36.219 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3922995200 bytes 2026-02-20T22:11:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3961933824 bytes 2026-02-20T22:11:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3979786240 bytes 2026-02-20T22:11:36.363 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 4012503040 bytes 2026-02-20T22:11:36.402 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 4031690752 bytes 2026-02-20T22:11:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 4044961792 bytes 2026-02-20T22:11:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 4087043072 bytes 2026-02-20T22:11:36.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 4117456896 bytes 2026-02-20T22:11:36.569 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 4152940544 bytes 2026-02-20T22:11:36.617 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 4196942848 bytes 2026-02-20T22:11:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 4325819392 bytes 2026-02-20T22:11:36.743 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 4347764736 bytes 2026-02-20T22:11:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4416247808 bytes 2026-02-20T22:11:36.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4445142016 bytes 2026-02-20T22:11:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4487830528 bytes 2026-02-20T22:11:36.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4495925248 bytes 2026-02-20T22:11:36.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4553237504 bytes 2026-02-20T22:11:37.043 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4626759680 bytes 2026-02-20T22:11:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4652377088 bytes 2026-02-20T22:11:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4726625280 bytes 2026-02-20T22:11:37.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4786570240 bytes 2026-02-20T22:11:37.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4819848192 bytes 2026-02-20T22:11:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4860999680 bytes 2026-02-20T22:11:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4877868032 bytes 2026-02-20T22:11:37.383 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4899292160 bytes 2026-02-20T22:11:37.424 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4930119680 bytes 2026-02-20T22:11:37.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4961750016 bytes 2026-02-20T22:11:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4970617856 bytes 2026-02-20T22:11:37.539 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4975087616 bytes 2026-02-20T22:11:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 5042336768 bytes 2026-02-20T22:11:37.640 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 5073898496 bytes 2026-02-20T22:11:37.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 5132588032 bytes 2026-02-20T22:11:37.741 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 5163752448 bytes 2026-02-20T22:11:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 5209218048 bytes 2026-02-20T22:11:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 5232874496 bytes 2026-02-20T22:11:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 5258739712 bytes 2026-02-20T22:11:37.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 5268217856 bytes 2026-02-20T22:11:37.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 5306882048 bytes 2026-02-20T22:11:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5313495040 bytes 2026-02-20T22:11:38.039 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5332640768 bytes 2026-02-20T22:11:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5374926848 bytes 2026-02-20T22:11:38.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5446250496 bytes 2026-02-20T22:11:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5525884928 bytes 2026-02-20T22:11:38.267 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5589458944 bytes 2026-02-20T22:11:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5629937664 bytes 2026-02-20T22:11:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5703993344 bytes 2026-02-20T22:11:38.407 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5704592384 bytes 2026-02-20T22:11:38.453 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5742446592 bytes 2026-02-20T22:11:38.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5760435200 bytes 2026-02-20T22:11:38.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5793196032 bytes 2026-02-20T22:11:38.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5827402752 bytes 2026-02-20T22:11:38.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5877361664 bytes 2026-02-20T22:11:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5892200448 bytes 2026-02-20T22:11:38.722 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5940339712 bytes 2026-02-20T22:11:38.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5942726656 bytes 2026-02-20T22:11:38.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5983039488 bytes 2026-02-20T22:11:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 6032399360 bytes 2026-02-20T22:11:38.922 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 6112621568 bytes 2026-02-20T22:11:38.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 6183798784 bytes 2026-02-20T22:11:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 6204176384 bytes 2026-02-20T22:11:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 6204780544 bytes 2026-02-20T22:11:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 6218182656 bytes 2026-02-20T22:11:39.136 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 6244245504 bytes 2026-02-20T22:11:39.196 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 6337646592 bytes 2026-02-20T22:11:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 6377691136 bytes 2026-02-20T22:11:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6475932672 bytes 2026-02-20T22:11:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6519202816 bytes 2026-02-20T22:11:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6553142272 bytes 2026-02-20T22:11:39.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6606748672 bytes 2026-02-20T22:11:39.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6677929984 bytes 2026-02-20T22:11:39.580 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6713705472 bytes 2026-02-20T22:11:39.615 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6730100736 bytes 2026-02-20T22:11:39.651 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6743936000 bytes 2026-02-20T22:11:39.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6802165760 bytes 2026-02-20T22:11:39.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6849084416 bytes 2026-02-20T22:11:39.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6896246784 bytes 2026-02-20T22:11:39.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6896835584 bytes 2026-02-20T22:11:39.873 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6920235008 bytes 2026-02-20T22:11:39.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6938077184 bytes 2026-02-20T22:11:39.970 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 7004283904 bytes 2026-02-20T22:11:40.044 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 7102073856 bytes 2026-02-20T22:11:40.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 7152695296 bytes 2026-02-20T22:11:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 7189916672 bytes 2026-02-20T22:11:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 7190499328 bytes 2026-02-20T22:11:40.212 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 7215796224 bytes 2026-02-20T22:11:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7245152256 bytes 2026-02-20T22:11:40.289 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7249318912 bytes 2026-02-20T22:11:40.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7298089984 bytes 2026-02-20T22:11:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7344822272 bytes 2026-02-20T22:11:40.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7381533696 bytes 2026-02-20T22:11:40.486 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7422279680 bytes 2026-02-20T22:11:40.527 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7442463744 bytes 2026-02-20T22:11:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7513485312 bytes 2026-02-20T22:11:40.633 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7543906304 bytes 2026-02-20T22:11:40.696 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7622206464 bytes 2026-02-20T22:11:40.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7709137920 bytes 2026-02-20T22:11:40.797 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7724201984 bytes 2026-02-20T22:11:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7804613632 bytes 2026-02-20T22:11:40.913 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7860753408 bytes 2026-02-20T22:11:40.951 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7867418624 bytes 2026-02-20T22:11:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7903137792 bytes 2026-02-20T22:11:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7925811200 bytes 2026-02-20T22:11:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7943919616 bytes 2026-02-20T22:11:41.108 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7946782720 bytes 2026-02-20T22:11:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 8003928064 bytes 2026-02-20T22:11:41.217 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 8078457856 bytes 2026-02-20T22:11:41.274 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 8152156160 bytes 2026-02-20T22:11:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 8181223424 bytes 2026-02-20T22:11:41.381 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 8226671616 bytes 2026-02-20T22:11:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 8273472512 bytes 2026-02-20T22:11:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 8349965312 bytes 2026-02-20T22:11:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 8362903552 bytes 2026-02-20T22:11:41.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 8405660672 bytes 2026-02-20T22:11:41.625 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 8454737920 bytes 2026-02-20T22:11:41.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 8468088832 bytes 2026-02-20T22:11:41.716 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8530832384 bytes 2026-02-20T22:11:41.769 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8585990144 bytes 2026-02-20T22:11:41.814 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8621932544 bytes 2026-02-20T22:11:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8692760576 bytes 2026-02-20T22:11:41.920 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8717412352 bytes 2026-02-20T22:11:41.968 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8751551488 bytes 2026-02-20T22:11:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8798785536 bytes 2026-02-20T22:11:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8896773120 bytes 2026-02-20T22:11:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8966048768 bytes 2026-02-20T22:11:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8995336192 bytes 2026-02-20T22:11:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 9032881152 bytes 2026-02-20T22:11:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 9062926336 bytes 2026-02-20T22:11:42.321 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 9091542016 bytes 2026-02-20T22:11:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 9151386624 bytes 2026-02-20T22:11:42.416 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 9161139200 bytes 2026-02-20T22:11:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 9217860608 bytes 2026-02-20T22:11:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 9283932160 bytes 2026-02-20T22:11:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 9337006080 bytes 2026-02-20T22:11:42.631 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9379124224 bytes 2026-02-20T22:11:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9450907648 bytes 2026-02-20T22:11:42.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9487551488 bytes 2026-02-20T22:11:42.776 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9515867136 bytes 2026-02-20T22:11:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9516463104 bytes 2026-02-20T22:11:42.851 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9538814976 bytes 2026-02-20T22:11:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9573931008 bytes 2026-02-20T22:11:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9596514304 bytes 2026-02-20T22:11:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9602305024 bytes 2026-02-20T22:11:43.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9644850176 bytes 2026-02-20T22:11:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9672406016 bytes 2026-02-20T22:11:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9708220416 bytes 2026-02-20T22:11:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9753796608 bytes 2026-02-20T22:11:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9754407936 bytes 2026-02-20T22:11:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9838526464 bytes 2026-02-20T22:11:43.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9877999616 bytes 2026-02-20T22:11:43.377 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9985142784 bytes 2026-02-20T22:11:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9996845056 bytes 2026-02-20T22:11:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 10014605312 bytes 2026-02-20T22:11:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 10107062272 bytes 2026-02-20T22:11:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 10146199552 bytes 2026-02-20T22:11:43.607 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 10174268416 bytes 2026-02-20T22:11:43.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 10233534464 bytes 2026-02-20T22:11:43.705 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 10264877056 bytes 2026-02-20T22:11:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 10345382912 bytes 2026-02-20T22:11:43.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10427404288 bytes 2026-02-20T22:11:43.887 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 10482908160 bytes 2026-02-20T22:11:43.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:11:43.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:11:43.894 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:11:43.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T22:11:43.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:11:43.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:11:43.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_9 2026-02-20T22:11:43.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T22:11:43.895 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:11:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b0d0d392ec 2026-02-20T22:11:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:11:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:11:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_9 present 10b0d0d392ec 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b0d0d392ec 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:11:44.025 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:11:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_9 2026-02-20T22:11:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T22:11:44.026 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:11:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b0d0d392ec 2026-02-20T22:11:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b0d0d392ec 2026-02-20T22:11:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b0d0d392ec -o 10b0d0d392ec = 10b0d0d392ec 2026-02-20T22:11:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:11:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:11:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:11:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_9 2026-02-20T22:11:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:11:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:11:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:11:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2026-02-20T22:11:46.115 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:11:46.111+0000 7fc7a2858640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55e036bccb60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:11:46.116 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:11:46.111+0000 7fc7a9065640 -1 rbd::mirror::ImageReplayer: 0x55e036e84280 [3/2ee8fc72-8ddb-4980-a83f-4c17100a038c] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:11:55.550 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5586836732918782) > weight (0.25) 2026-02-20T22:11:55.551 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.2852584090415131) > weight (0.25) 2026-02-20T22:11:55.551 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.587901390596704) > weight (0.25) 2026-02-20T22:11:55.551 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.39521000223463765) > weight (0.25) 2026-02-20T22:11:55.551 INFO:tasks.rbd_mirror_thrash:waiting for 99.09976197874964 secs before thrashing 2026-02-20T22:12:03.266 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9821321543665062) > weight (0.25) 2026-02-20T22:12:03.267 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4699985740103738) > weight (0.25) 2026-02-20T22:12:03.267 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5183719627651195) > weight (0.25) 2026-02-20T22:12:03.267 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.8076224896014387) > weight (0.25) 2026-02-20T22:12:03.267 INFO:tasks.rbd_mirror_thrash:waiting for 99.60108917375793 secs before thrashing 2026-02-20T22:12:14.783 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:12:14.778+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7dab5c00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=193 2026-02-20T22:12:14.784 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:12:14.778+0000 7f74000ec640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a7dab5c00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=196, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=c1, last_copied_object_number=128, snap_seqs={193=18446744073709551614}] 2026-02-20T22:12:14.788 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:12:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:12:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:12:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_9 2026-02-20T22:12:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T22:12:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:12:14.823 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-20T22:12:14.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:12:14.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:12:14.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:12:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-20T22:12:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:12:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:14.928 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_9 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T22:12:14.929 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:12:14.950 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-20T22:12:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:12:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:12:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:12:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:12:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_9 2026-02-20T22:12:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:12:14.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2026-02-20T22:12:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-20T22:12:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:12:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:12:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:12:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_9 2026-02-20T22:12:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:14.986 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_9 2026-02-20T22:12:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-20T22:12:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-20T22:12:15.168 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:12:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:12:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for i in `seq 1 10` 2026-02-20T22:12:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image=image_10 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ create_image cluster2 mirror image_10 512M 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local size=128 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -n 512M ']' 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ size=512M 2026-02-20T22:12:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ shift 2026-02-20T22:12:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,object-map,fast-diff mirror/image_10 2026-02-20T22:12:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ enable_mirror cluster2 mirror image_10 2026-02-20T22:12:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local mode=snapshot 2026-02-20T22:12:15.206 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 snapshot 2026-02-20T22:12:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring enabled 2026-02-20T22:12:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-02-20T22:12:15.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:rbd image 'image_10': 2026-02-20T22:12:15.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: size 512 MiB in 128 objects 2026-02-20T22:12:15.831 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: order 22 (4 MiB objects) 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: snapshot_count: 1 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: id: 19e3fea757a3 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: block_name_prefix: rbd_data.19e3fea757a3 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: format: 2 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: features: layering, exclusive-lock, object-map, fast-diff 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: op_features: 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: flags: 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: create_timestamp: Fri Feb 20 22:12:15 2026 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: access_timestamp: Fri Feb 20 22:12:15 2026 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: modify_timestamp: Fri Feb 20 22:12:15 2026 2026-02-20T22:12:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring state: enabled 2026-02-20T22:12:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring mode: snapshot 2026-02-20T22:12:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring global id: 20e45c09-bd75-447d-9ec1-1658700f51ae 2026-02-20T22:12:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout: mirroring primary: true 2026-02-20T22:12:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-20T22:12:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T22:12:15.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local duration=13 2026-02-20T22:12:15.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:12:15.836 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-20T22:12:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:100 IOs, 52030464 bytes 2026-02-20T22:12:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:200 IOs, 99687424 bytes 2026-02-20T22:12:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:300 IOs, 103228416 bytes 2026-02-20T22:12:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:400 IOs, 125417472 bytes 2026-02-20T22:12:16.174 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:500 IOs, 184055808 bytes 2026-02-20T22:12:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:600 IOs, 232035328 bytes 2026-02-20T22:12:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:700 IOs, 261103616 bytes 2026-02-20T22:12:16.308 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:800 IOs, 297886720 bytes 2026-02-20T22:12:16.365 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:900 IOs, 355805184 bytes 2026-02-20T22:12:16.404 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1000 IOs, 381751296 bytes 2026-02-20T22:12:16.464 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1100 IOs, 453997568 bytes 2026-02-20T22:12:16.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1200 IOs, 488942592 bytes 2026-02-20T22:12:16.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1300 IOs, 560047104 bytes 2026-02-20T22:12:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1400 IOs, 560670720 bytes 2026-02-20T22:12:16.643 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1500 IOs, 582436864 bytes 2026-02-20T22:12:16.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1600 IOs, 648092672 bytes 2026-02-20T22:12:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1700 IOs, 673770496 bytes 2026-02-20T22:12:16.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1800 IOs, 740098048 bytes 2026-02-20T22:12:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:1900 IOs, 765637632 bytes 2026-02-20T22:12:16.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2000 IOs, 817513472 bytes 2026-02-20T22:12:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2100 IOs, 872803328 bytes 2026-02-20T22:12:16.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2200 IOs, 898703360 bytes 2026-02-20T22:12:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2300 IOs, 974007296 bytes 2026-02-20T22:12:17.085 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2400 IOs, 979673088 bytes 2026-02-20T22:12:17.120 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2500 IOs, 987433984 bytes 2026-02-20T22:12:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2600 IOs, 990111744 bytes 2026-02-20T22:12:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2700 IOs, 1010249728 bytes 2026-02-20T22:12:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2800 IOs, 1063506944 bytes 2026-02-20T22:12:17.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:2900 IOs, 1096297472 bytes 2026-02-20T22:12:17.353 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3000 IOs, 1193668608 bytes 2026-02-20T22:12:17.392 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3100 IOs, 1204877312 bytes 2026-02-20T22:12:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3200 IOs, 1265391616 bytes 2026-02-20T22:12:17.490 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3300 IOs, 1292779520 bytes 2026-02-20T22:12:17.535 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3400 IOs, 1324299264 bytes 2026-02-20T22:12:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3500 IOs, 1374919680 bytes 2026-02-20T22:12:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3600 IOs, 1397292032 bytes 2026-02-20T22:12:17.699 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3700 IOs, 1503056896 bytes 2026-02-20T22:12:17.758 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3800 IOs, 1578333184 bytes 2026-02-20T22:12:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:3900 IOs, 1625719808 bytes 2026-02-20T22:12:17.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4000 IOs, 1676418048 bytes 2026-02-20T22:12:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4100 IOs, 1712677888 bytes 2026-02-20T22:12:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4200 IOs, 1745311744 bytes 2026-02-20T22:12:17.982 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4300 IOs, 1768425472 bytes 2026-02-20T22:12:18.030 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4400 IOs, 1788016640 bytes 2026-02-20T22:12:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4500 IOs, 1829272576 bytes 2026-02-20T22:12:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4600 IOs, 1887650816 bytes 2026-02-20T22:12:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4700 IOs, 1923843072 bytes 2026-02-20T22:12:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4800 IOs, 1954398208 bytes 2026-02-20T22:12:18.270 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:4900 IOs, 1991062528 bytes 2026-02-20T22:12:18.320 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5000 IOs, 2040410112 bytes 2026-02-20T22:12:18.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5100 IOs, 2097108992 bytes 2026-02-20T22:12:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5200 IOs, 2116484096 bytes 2026-02-20T22:12:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5300 IOs, 2193321984 bytes 2026-02-20T22:12:18.524 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5400 IOs, 2231100416 bytes 2026-02-20T22:12:18.566 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5500 IOs, 2260966400 bytes 2026-02-20T22:12:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5600 IOs, 2290260992 bytes 2026-02-20T22:12:18.660 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5700 IOs, 2342370304 bytes 2026-02-20T22:12:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5800 IOs, 2402851840 bytes 2026-02-20T22:12:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:5900 IOs, 2465856512 bytes 2026-02-20T22:12:18.816 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6000 IOs, 2495822848 bytes 2026-02-20T22:12:18.855 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6100 IOs, 2520142848 bytes 2026-02-20T22:12:18.912 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6200 IOs, 2568187904 bytes 2026-02-20T22:12:18.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6300 IOs, 2608205824 bytes 2026-02-20T22:12:19.005 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6400 IOs, 2644695040 bytes 2026-02-20T22:12:19.073 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6500 IOs, 2727455744 bytes 2026-02-20T22:12:19.114 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6600 IOs, 2748563456 bytes 2026-02-20T22:12:19.171 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6700 IOs, 2815666176 bytes 2026-02-20T22:12:19.218 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6800 IOs, 2855069696 bytes 2026-02-20T22:12:19.251 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:6900 IOs, 2861268992 bytes 2026-02-20T22:12:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7000 IOs, 2889564160 bytes 2026-02-20T22:12:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7100 IOs, 2940495872 bytes 2026-02-20T22:12:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7200 IOs, 2990377984 bytes 2026-02-20T22:12:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7300 IOs, 3038712832 bytes 2026-02-20T22:12:19.494 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7400 IOs, 3085612032 bytes 2026-02-20T22:12:19.532 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7500 IOs, 3124186112 bytes 2026-02-20T22:12:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7600 IOs, 3202411520 bytes 2026-02-20T22:12:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7700 IOs, 3312226304 bytes 2026-02-20T22:12:19.737 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7800 IOs, 3380088832 bytes 2026-02-20T22:12:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:7900 IOs, 3442624512 bytes 2026-02-20T22:12:19.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8000 IOs, 3485386752 bytes 2026-02-20T22:12:19.884 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8100 IOs, 3508551680 bytes 2026-02-20T22:12:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8200 IOs, 3526504448 bytes 2026-02-20T22:12:19.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8300 IOs, 3527503872 bytes 2026-02-20T22:12:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8400 IOs, 3572184064 bytes 2026-02-20T22:12:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8500 IOs, 3635648512 bytes 2026-02-20T22:12:20.095 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8600 IOs, 3636264960 bytes 2026-02-20T22:12:20.131 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8700 IOs, 3649106944 bytes 2026-02-20T22:12:20.178 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8800 IOs, 3677324288 bytes 2026-02-20T22:12:20.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:8900 IOs, 3749586944 bytes 2026-02-20T22:12:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9000 IOs, 3772395520 bytes 2026-02-20T22:12:20.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9100 IOs, 3801439232 bytes 2026-02-20T22:12:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9200 IOs, 3837221888 bytes 2026-02-20T22:12:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9300 IOs, 3837849600 bytes 2026-02-20T22:12:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9400 IOs, 3942857728 bytes 2026-02-20T22:12:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9500 IOs, 3982867456 bytes 2026-02-20T22:12:20.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9600 IOs, 4022045696 bytes 2026-02-20T22:12:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9700 IOs, 4042283008 bytes 2026-02-20T22:12:20.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9800 IOs, 4118744064 bytes 2026-02-20T22:12:20.725 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:9900 IOs, 4136638464 bytes 2026-02-20T22:12:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10000 IOs, 4192892928 bytes 2026-02-20T22:12:20.822 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10100 IOs, 4219439104 bytes 2026-02-20T22:12:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10200 IOs, 4247191552 bytes 2026-02-20T22:12:20.908 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10300 IOs, 4278233088 bytes 2026-02-20T22:12:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10400 IOs, 4316374016 bytes 2026-02-20T22:12:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10500 IOs, 4344224768 bytes 2026-02-20T22:12:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10600 IOs, 4397258752 bytes 2026-02-20T22:12:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10700 IOs, 4444751872 bytes 2026-02-20T22:12:21.152 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10800 IOs, 4481006592 bytes 2026-02-20T22:12:21.201 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:10900 IOs, 4526527488 bytes 2026-02-20T22:12:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11000 IOs, 4527157248 bytes 2026-02-20T22:12:21.278 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11100 IOs, 4567936000 bytes 2026-02-20T22:12:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11200 IOs, 4613031936 bytes 2026-02-20T22:12:21.369 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11300 IOs, 4633116672 bytes 2026-02-20T22:12:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11400 IOs, 4660793344 bytes 2026-02-20T22:12:21.472 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11500 IOs, 4724694016 bytes 2026-02-20T22:12:21.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11600 IOs, 4783560704 bytes 2026-02-20T22:12:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11700 IOs, 4834796544 bytes 2026-02-20T22:12:21.634 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11800 IOs, 4889079808 bytes 2026-02-20T22:12:21.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:11900 IOs, 4917403648 bytes 2026-02-20T22:12:21.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12000 IOs, 4970723328 bytes 2026-02-20T22:12:21.771 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12100 IOs, 4986943488 bytes 2026-02-20T22:12:21.809 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12200 IOs, 5010756608 bytes 2026-02-20T22:12:21.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12300 IOs, 5044252672 bytes 2026-02-20T22:12:21.909 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12400 IOs, 5096300544 bytes 2026-02-20T22:12:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12500 IOs, 5113377792 bytes 2026-02-20T22:12:22.009 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12600 IOs, 5188320256 bytes 2026-02-20T22:12:22.072 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12700 IOs, 5236746240 bytes 2026-02-20T22:12:22.104 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12800 IOs, 5237383168 bytes 2026-02-20T22:12:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:12900 IOs, 5263706112 bytes 2026-02-20T22:12:22.190 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13000 IOs, 5298092032 bytes 2026-02-20T22:12:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13100 IOs, 5348953088 bytes 2026-02-20T22:12:22.285 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13200 IOs, 5374312448 bytes 2026-02-20T22:12:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13300 IOs, 5386989568 bytes 2026-02-20T22:12:22.360 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13400 IOs, 5424064512 bytes 2026-02-20T22:12:22.405 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13500 IOs, 5438655488 bytes 2026-02-20T22:12:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13600 IOs, 5520323584 bytes 2026-02-20T22:12:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13700 IOs, 5539874816 bytes 2026-02-20T22:12:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13800 IOs, 5609029632 bytes 2026-02-20T22:12:22.606 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:13900 IOs, 5618016256 bytes 2026-02-20T22:12:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14000 IOs, 5643927552 bytes 2026-02-20T22:12:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14100 IOs, 5691501568 bytes 2026-02-20T22:12:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14200 IOs, 5733536768 bytes 2026-02-20T22:12:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14300 IOs, 5775285248 bytes 2026-02-20T22:12:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14400 IOs, 5827566592 bytes 2026-02-20T22:12:22.890 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14500 IOs, 5866198016 bytes 2026-02-20T22:12:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14600 IOs, 5909682176 bytes 2026-02-20T22:12:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14700 IOs, 5959711744 bytes 2026-02-20T22:12:23.028 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14800 IOs, 5978831872 bytes 2026-02-20T22:12:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:14900 IOs, 6037960704 bytes 2026-02-20T22:12:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15000 IOs, 6117705728 bytes 2026-02-20T22:12:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15100 IOs, 6163610624 bytes 2026-02-20T22:12:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15200 IOs, 6193006592 bytes 2026-02-20T22:12:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15300 IOs, 6239850496 bytes 2026-02-20T22:12:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15400 IOs, 6352284672 bytes 2026-02-20T22:12:23.438 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15500 IOs, 6454181888 bytes 2026-02-20T22:12:23.495 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15600 IOs, 6497901568 bytes 2026-02-20T22:12:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15700 IOs, 6526927872 bytes 2026-02-20T22:12:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15800 IOs, 6579444736 bytes 2026-02-20T22:12:23.635 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:15900 IOs, 6615026688 bytes 2026-02-20T22:12:23.692 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16000 IOs, 6679731200 bytes 2026-02-20T22:12:23.736 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16100 IOs, 6709138432 bytes 2026-02-20T22:12:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16200 IOs, 6715038720 bytes 2026-02-20T22:12:23.804 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16300 IOs, 6739500032 bytes 2026-02-20T22:12:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16400 IOs, 6799416320 bytes 2026-02-20T22:12:23.910 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16500 IOs, 6848770048 bytes 2026-02-20T22:12:23.967 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16600 IOs, 6902178816 bytes 2026-02-20T22:12:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16700 IOs, 6915614720 bytes 2026-02-20T22:12:24.063 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16800 IOs, 6991104000 bytes 2026-02-20T22:12:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:16900 IOs, 7022756864 bytes 2026-02-20T22:12:24.155 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17000 IOs, 7043892224 bytes 2026-02-20T22:12:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17100 IOs, 7073862656 bytes 2026-02-20T22:12:24.246 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17200 IOs, 7120652288 bytes 2026-02-20T22:12:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17300 IOs, 7141568512 bytes 2026-02-20T22:12:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17400 IOs, 7167415296 bytes 2026-02-20T22:12:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17500 IOs, 7198929920 bytes 2026-02-20T22:12:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17600 IOs, 7229329408 bytes 2026-02-20T22:12:24.481 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17700 IOs, 7310410752 bytes 2026-02-20T22:12:24.529 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17800 IOs, 7354292224 bytes 2026-02-20T22:12:24.563 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:17900 IOs, 7356498944 bytes 2026-02-20T22:12:24.612 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18000 IOs, 7402084352 bytes 2026-02-20T22:12:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18100 IOs, 7444716544 bytes 2026-02-20T22:12:24.695 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18200 IOs, 7462219776 bytes 2026-02-20T22:12:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18300 IOs, 7490505728 bytes 2026-02-20T22:12:24.792 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18400 IOs, 7549278208 bytes 2026-02-20T22:12:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18500 IOs, 7642603520 bytes 2026-02-20T22:12:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18600 IOs, 7666796544 bytes 2026-02-20T22:12:24.939 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18700 IOs, 7694353408 bytes 2026-02-20T22:12:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18800 IOs, 7767598080 bytes 2026-02-20T22:12:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:18900 IOs, 7777666048 bytes 2026-02-20T22:12:25.069 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19000 IOs, 7778255872 bytes 2026-02-20T22:12:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19100 IOs, 7805701120 bytes 2026-02-20T22:12:25.145 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19200 IOs, 7814810624 bytes 2026-02-20T22:12:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19300 IOs, 7848756224 bytes 2026-02-20T22:12:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19400 IOs, 7858480128 bytes 2026-02-20T22:12:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19500 IOs, 7875589120 bytes 2026-02-20T22:12:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19600 IOs, 7895740416 bytes 2026-02-20T22:12:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19700 IOs, 7940447232 bytes 2026-02-20T22:12:25.400 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19800 IOs, 7989560320 bytes 2026-02-20T22:12:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:19900 IOs, 8005423104 bytes 2026-02-20T22:12:25.488 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20000 IOs, 8025991168 bytes 2026-02-20T22:12:25.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20100 IOs, 8096913408 bytes 2026-02-20T22:12:25.597 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20200 IOs, 8153522176 bytes 2026-02-20T22:12:25.649 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20300 IOs, 8205130752 bytes 2026-02-20T22:12:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20400 IOs, 8222021632 bytes 2026-02-20T22:12:25.727 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20500 IOs, 8247488512 bytes 2026-02-20T22:12:25.817 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20600 IOs, 8391468032 bytes 2026-02-20T22:12:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20700 IOs, 8497505280 bytes 2026-02-20T22:12:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20800 IOs, 8572498944 bytes 2026-02-20T22:12:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:20900 IOs, 8631524352 bytes 2026-02-20T22:12:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21000 IOs, 8695174144 bytes 2026-02-20T22:12:26.115 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21100 IOs, 8724786176 bytes 2026-02-20T22:12:26.166 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21200 IOs, 8786797568 bytes 2026-02-20T22:12:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21300 IOs, 8829683712 bytes 2026-02-20T22:12:26.482 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21400 IOs, 8857237504 bytes 2026-02-20T22:12:26.533 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21500 IOs, 8901482496 bytes 2026-02-20T22:12:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21600 IOs, 8968868864 bytes 2026-02-20T22:12:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21700 IOs, 9015542784 bytes 2026-02-20T22:12:26.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21800 IOs, 9096101888 bytes 2026-02-20T22:12:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:21900 IOs, 9115336704 bytes 2026-02-20T22:12:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22000 IOs, 9128966144 bytes 2026-02-20T22:12:26.829 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22100 IOs, 9179440128 bytes 2026-02-20T22:12:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22200 IOs, 9192075264 bytes 2026-02-20T22:12:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22300 IOs, 9234990080 bytes 2026-02-20T22:12:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22400 IOs, 9249940480 bytes 2026-02-20T22:12:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22500 IOs, 9289577472 bytes 2026-02-20T22:12:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22600 IOs, 9294382080 bytes 2026-02-20T22:12:27.107 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22700 IOs, 9378028544 bytes 2026-02-20T22:12:27.149 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22800 IOs, 9401779200 bytes 2026-02-20T22:12:27.213 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:22900 IOs, 9480935424 bytes 2026-02-20T22:12:27.259 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23000 IOs, 9522569216 bytes 2026-02-20T22:12:27.305 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23100 IOs, 9559485440 bytes 2026-02-20T22:12:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23200 IOs, 9579679744 bytes 2026-02-20T22:12:27.398 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23300 IOs, 9637926912 bytes 2026-02-20T22:12:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23400 IOs, 9651251200 bytes 2026-02-20T22:12:27.544 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23500 IOs, 9741028352 bytes 2026-02-20T22:12:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23600 IOs, 9810289664 bytes 2026-02-20T22:12:27.646 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23700 IOs, 9824776192 bytes 2026-02-20T22:12:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23800 IOs, 9861884928 bytes 2026-02-20T22:12:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:23900 IOs, 9924414464 bytes 2026-02-20T22:12:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24000 IOs, 9983726592 bytes 2026-02-20T22:12:27.863 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24100 IOs, 10031527936 bytes 2026-02-20T22:12:27.905 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24200 IOs, 10054078464 bytes 2026-02-20T22:12:27.953 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24300 IOs, 10093432832 bytes 2026-02-20T22:12:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24400 IOs, 10157524992 bytes 2026-02-20T22:12:28.065 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24500 IOs, 10212101120 bytes 2026-02-20T22:12:28.110 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24600 IOs, 10246542336 bytes 2026-02-20T22:12:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24700 IOs, 10299097088 bytes 2026-02-20T22:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24800 IOs, 10312639488 bytes 2026-02-20T22:12:28.250 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:24900 IOs, 10344735744 bytes 2026-02-20T22:12:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25000 IOs, 10375554048 bytes 2026-02-20T22:12:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25100 IOs, 10440125440 bytes 2026-02-20T22:12:28.401 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25200 IOs, 10504800256 bytes 2026-02-20T22:12:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25300 IOs, 10544706560 bytes 2026-02-20T22:12:28.542 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25400 IOs, 10634290176 bytes 2026-02-20T22:12:28.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25500 IOs, 10634891264 bytes 2026-02-20T22:12:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25600 IOs, 10674643968 bytes 2026-02-20T22:12:28.662 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25700 IOs, 10695460864 bytes 2026-02-20T22:12:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25800 IOs, 10753483776 bytes 2026-02-20T22:12:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:25900 IOs, 10843166720 bytes 2026-02-20T22:12:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:26000 IOs, 10879975424 bytes 2026-02-20T22:12:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=124 2026-02-20T22:12:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:12:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T22:12:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:12:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2026-02-20T22:12:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:12:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=present 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_10 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T22:12:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:12:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id=10b046b2ecd8 2026-02-20T22:12:28.874 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:12:28.875 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:12:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_10 present 10b046b2ecd8 2026-02-20T22:12:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=present 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id=10b046b2ecd8 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_10 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T22:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:12:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id=10b046b2ecd8 2026-02-20T22:12:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 10b046b2ecd8 2026-02-20T22:12:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -z 10b046b2ecd8 -o 10b046b2ecd8 = 10b046b2ecd8 2026-02-20T22:12:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_state=present 2026-02-20T22:12:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test present = present 2026-02-20T22:12:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:12:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ disable_mirror cluster2 mirror image_10 2026-02-20T22:12:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2026-02-20T22:12:46.116 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:12:46.109+0000 7fc7a2858640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x55e036d57860 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T22:12:46.116 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:12:46.109+0000 7fc7a9065640 -1 rbd::mirror::ImageReplayer: 0x55e036e84000 [3/20e45c09-bd75-447d-9ec1-1658700f51ae] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T22:12:59.784 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:12:59.777+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a79ac1c00 scan_remote_mirror_snapshots: failed to locate remote start snapshot: snap_id=194 2026-02-20T22:12:59.784 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:12:59.777+0000 7f74008ed640 -1 rbd::mirror::image_replayer::snapshot::Replayer: 0x564a79ac1c00 scan_remote_mirror_snapshots: split-brain detected: failed to find matching non-primary snapshot in remote image: local_snap_id_start=197, local_snap_ns=[mirror state=non-primary, complete=1, mirror_peer_uuids=, primary_mirror_uuid=aa40d3f4-0618-4c9d-a0d0-5b630053911d, primary_snap_id=c2, last_copied_object_number=128, snap_seqs={194=18446744073709551614}] 2026-02-20T22:12:59.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:Mirroring disabled 2026-02-20T22:12:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-20T22:12:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local state=deleted 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local s 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_10 2026-02-20T22:12:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T22:12:59.803 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:12:59.826 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-20T22:12:59.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ image_id= 2026-02-20T22:12:59.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T22:12:59.830 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0.1 2026-02-20T22:12:59.931 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local test_state=deleted 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image_id= 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_state=deleted 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local current_image_id 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:12:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:12:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local image=image_10 2026-02-20T22:12:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T22:12:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T22:12:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-20T22:12:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ current_image_id= 2026-02-20T22:12:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n '' 2026-02-20T22:12:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test deleted = deleted 2026-02-20T22:12:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:12:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ purge_snapshots cluster2 mirror image_10 2026-02-20T22:12:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2026-02-20T22:12:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-20T22:12:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T22:12:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:12:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ remove_image cluster2 mirror image_10 2026-02-20T22:12:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:12:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:12:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local image=image_10 2026-02-20T22:12:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-20T22:13:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-20T22:13:00.567 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T22:13:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-20T22:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ tee -a /tmp/tmp.2SC0OFePO0/rbd-mirror.test.log 2026-02-20T22:13:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ date '+%F %T' 2026-02-20T22:13:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo 2026-02-20 22:13:00 TEST: check if removed 'images'\''' OMAP are removed 2026-02-20T22:13:00.574 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:2026-02-20 22:13:00 TEST: check if removed images' OMAP are removed 2026-02-20T22:13:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-20T22:13:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:13:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:13:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-20T22:13:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:13:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:13:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local obj_name=rbd_mirroring 2026-02-20T22:13:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local filter=status_global 2026-02-20T22:13:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T22:13:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:13:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:13:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-20T22:13:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirroring 2026-02-20T22:13:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local filter=status_global 2026-02-20T22:13:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -c status_global 2026-02-20T22:13:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-20T22:13:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirroring 2026-02-20T22:13:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-20T22:13:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 = 0 2026-02-20T22:13:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T22:13:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:13:00.601 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T22:13:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:13:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-20T22:13:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:13:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:13:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local obj_name=rbd_mirroring 2026-02-20T22:13:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local filter=image_ 2026-02-20T22:13:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T22:13:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirroring 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local filter=image_ 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirroring 2026-02-20T22:13:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-20T22:13:00.605 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -c image_ 2026-02-20T22:13:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 = 0 2026-02-20T22:13:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T22:13:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:13:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T22:13:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:13:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-20T22:13:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:13:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:13:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local obj_name=rbd_mirror_leader 2026-02-20T22:13:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local filter=image_map 2026-02-20T22:13:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T22:13:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:13:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:13:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T22:13:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T22:13:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local filter=image_map 2026-02-20T22:13:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T22:13:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -c image_map 2026-02-20T22:13:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T22:13:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T22:13:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 1 = 0 2026-02-20T22:13:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=1 2026-02-20T22:13:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:13:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 1 -eq 0 ']' 2026-02-20T22:13:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T22:13:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 1 2026-02-20T22:13:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:13:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T22:13:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T22:13:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local filter=image_map 2026-02-20T22:13:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T22:13:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -c image_map 2026-02-20T22:13:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T22:13:01.655 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T22:13:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 1 = 0 2026-02-20T22:13:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=1 2026-02-20T22:13:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:13:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 1 -eq 0 ']' 2026-02-20T22:13:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T22:13:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 2 2026-02-20T22:13:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:13:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T22:13:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T22:13:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local filter=image_map 2026-02-20T22:13:03.681 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T22:13:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster1 2026-02-20T22:13:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -c image_map 2026-02-20T22:13:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T22:13:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T22:13:03.707 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 = 0 2026-02-20T22:13:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T22:13:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:13:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T22:13:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:13:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-20T22:13:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:13:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:13:03.708 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-20T22:13:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:13:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:13:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local obj_name=rbd_mirroring 2026-02-20T22:13:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local filter=status_global 2026-02-20T22:13:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T22:13:03.709 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:13:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:13:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-20T22:13:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster2 2026-02-20T22:13:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirroring 2026-02-20T22:13:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local filter=status_global 2026-02-20T22:13:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-20T22:13:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster2 2026-02-20T22:13:03.711 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:03.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirroring 2026-02-20T22:13:03.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -c status_global 2026-02-20T22:13:03.712 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-20T22:13:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 = 0 2026-02-20T22:13:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T22:13:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:13:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T22:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-20T22:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local obj_name=rbd_mirroring 2026-02-20T22:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local filter=image_ 2026-02-20T22:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T22:13:03.731 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:13:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:13:03.732 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster2 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirroring 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local filter=image_ 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster2 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirroring 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -c image_ 2026-02-20T22:13:03.733 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 = 0 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local pool=mirror 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local obj_name=rbd_mirror_leader 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local filter=image_map 2026-02-20T22:13:03.754 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T22:13:03.755 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ sleep 0 2026-02-20T22:13:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:13:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-20T22:13:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster2 2026-02-20T22:13:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T22:13:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local filter=image_map 2026-02-20T22:13:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-20T22:13:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local cluster=cluster2 2026-02-20T22:13:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local pool=mirror 2026-02-20T22:13:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ grep -c image_map 2026-02-20T22:13:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T22:13:03.757 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T22:13:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test 0 = 0 2026-02-20T22:13:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ error_code=0 2026-02-20T22:13:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set -e 2026-02-20T22:13:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T22:13:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 0 2026-02-20T22:13:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ cleanup 0 2026-02-20T22:13:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local error_code=0 2026-02-20T22:13:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ set +e 2026-02-20T22:13:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -ne 0 ']' 2026-02-20T22:13:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -z '' ']' 2026-02-20T22:13:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster instance 2026-02-20T22:13:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T22:13:03.778 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-20T22:13:04.779 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:pool 'mirror' does not exist 2026-02-20T22:13:04.789 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T22:13:04.790 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-20T22:13:05.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f740d106640 -1 rbd::mirror::LeaderWatcher: 0x564a7a04b800 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-20T22:13:05.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f74068f9640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564a79ad4fc0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:05.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f74070fa640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564a7a142000 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:05.812 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f74070fa640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564a79ad5200 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:05.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f74070fa640 -1 rbd::mirror::PoolWatcher: 0x564a7a0ca000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:05.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f74068f9640 -1 rbd::mirror::PoolWatcher: 0x564a7a0caf00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:05.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f74070fa640 -1 rbd::mirror::PoolWatcher: 0x564a7a0cba40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:05.813 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f74008ed640 -1 rbd::mirror::PoolWatcher: 0x564a7832da40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:05.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f74008ed640 -1 rbd::mirror::PoolWatcher: 0x564a7a0ca780 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:05.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f74008ed640 -1 rbd::mirror::PoolWatcher: 0x564a7a0ca1e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:05.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f740d106640 -1 rbd::mirror::LeaderWatcher: 0x564a7a04b800 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-20T22:13:05.814 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:05.805+0000 7f740d106640 -1 rbd::mirror::LeaderWatcher: 0x564a7a04b800 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-20T22:13:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:pool 'mirror' does not exist 2026-02-20T22:13:05.879 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T22:13:05.880 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-20T22:13:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:pool 'mirror_parent' does not exist 2026-02-20T22:13:06.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ CEPH_ARGS= 2026-02-20T22:13:06.796 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-20T22:13:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:pool 'mirror_parent' does not exist 2026-02-20T22:13:07.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-20T22:13:07.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirrors cluster1 2026-02-20T22:13:07.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1 2026-02-20T22:13:07.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.849 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 0 3 2026-02-20T22:13:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:13:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirror cluster1:0 '' 2026-02-20T22:13:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1:0 2026-02-20T22:13:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T22:13:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T22:13:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:13:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirror cluster1:1 '' 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1:1 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirror cluster1:2 '' 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1:2 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:13:07.853 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirror cluster1:3 '' 2026-02-20T22:13:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster1:3 2026-02-20T22:13:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T22:13:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T22:13:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-20T22:13:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirrors cluster2 2026-02-20T22:13:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2 2026-02-20T22:13:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:++ seq 0 3 2026-02-20T22:13:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:13:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirror cluster2:0 '' 2026-02-20T22:13:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2:0 2026-02-20T22:13:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirror cluster2:1 '' 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2:1 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirror cluster2:2 '' 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2:2 2026-02-20T22:13:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ stop_mirror cluster2:3 '' 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local cluster=cluster2:3 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ local sig= 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test -n 1 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ return 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' -z 1 ']' 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ test '' = /tmp/tmp.2SC0OFePO0 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ rm -Rf /tmp/tmp.2SC0OFePO0 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T22:13:07.858 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ echo OK 2026-02-20T22:13:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial107.stdout:OK 2026-02-20T22:13:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial107.stderr:+ exit 0 2026-02-20T22:13:07.860 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-20T22:13:07.860 DEBUG:teuthology.orchestra.run.trial107:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-20T22:13:07.910 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2026-02-20T22:13:07.910 DEBUG:teuthology.orchestra.run.trial107:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-20T22:13:08.259 DEBUG:teuthology.parallel:result is None 2026-02-20T22:13:08.259 DEBUG:teuthology.orchestra.run.trial107:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-20T22:13:08.267 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-20T22:13:08.267 DEBUG:teuthology.orchestra.run.trial107:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T22:13:08.313 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-20T22:13:08.314 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-20T22:13:08.330 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-20T22:13:08.330 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 15 2026-02-20T22:13:08.330 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-20T22:13:08.331 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-20T22:13:08.347 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-20T22:13:08.347 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 13 2026-02-20T22:13:08.347 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-20T22:13:08.347 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T22:13:08.364 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2026-02-20T22:13:08.365 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:08.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.357+0000 7fc7a9866640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 52367) UID: 1000 2026-02-20T22:13:08.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.357+0000 7fc79483c640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55e036b9cb40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:08.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.357+0000 7fc79483c640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55e036b9cfc0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:08.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.357+0000 7fc7a2858640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55e036b9d8c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:08.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.357+0000 7fc7a2858640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55e036b9d680 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:08.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.357+0000 7fc7a2858640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x55e0374bd200 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:08.368 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79503d640 -1 rbd::mirror::PoolWatcher: 0x55e03558b2c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79503d640 -1 rbd::mirror::PoolWatcher: 0x55e036658d20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a2057640 -1 rbd::mirror::PoolWatcher: 0x55e036d914a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a2057640 -1 rbd::mirror::PoolWatcher: 0x55e036d903c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a2057640 -1 rbd::mirror::PoolWatcher: 0x55e036d910e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc78e02f640 -1 rbd::mirror::PoolWatcher: 0x55e036d38000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc78e830640 -1 rbd::mirror::PoolWatcher: 0x55e036d385a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79b84a640 -1 rbd::mirror::PoolWatcher: 0x55e0366592c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79b84a640 -1 rbd::mirror::PoolWatcher: 0x55e036d91860 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79b84a640 -1 rbd::mirror::PoolWatcher: 0x55e036d91a40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::InstanceWatcher: 0x55e036b9e380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6446 object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::InstanceWatcher: 0x55e03676e380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6446 object: (2) No such file or directory 2026-02-20T22:13:08.369 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a8864640 -1 rbd::mirror::InstanceWatcher: 0x55e0370cd500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6444 object: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::InstanceWatcher: 0x55e0370cd180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6444 object: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::InstanceWatcher: 0x55e03676e000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6444 object: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79483c640 -1 rbd::mirror::InstanceWatcher: 0x55e036b9e380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79483c640 -1 rbd::mirror::InstanceWatcher: 0x55e03676e380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a2858640 -1 rbd::mirror::InstanceWatcher: 0x55e0370cd500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a2057640 -1 rbd::mirror::InstanceWatcher: 0x55e03676e000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a2057640 -1 rbd::mirror::InstanceWatcher: 0x55e0370cd180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc78e02f640 -1 rbd::mirror::MirrorStatusUpdater 0x55e036714c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc78e02f640 -1 rbd::mirror::MirrorStatusUpdater 0x55e036714900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::NamespaceReplayer: 0x55e036717180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::NamespaceReplayer: 0x55e036717340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.370 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79483c640 -1 rbd::mirror::MirrorStatusUpdater 0x55e0369ca900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.371 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79483c640 -1 rbd::mirror::MirrorStatusUpdater 0x55e0369caa20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.371 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a8864640 -1 rbd::mirror::NamespaceReplayer: 0x55e036717180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.371 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a8864640 -1 rbd::mirror::NamespaceReplayer: 0x55e036717340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.371 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a8864640 -1 rbd::mirror::LeaderWatcher: 0x55e0369f7b00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-20T22:13:08.371 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a8864640 -1 rbd::mirror::LeaderWatcher: 0x55e0369f7b00 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-20T22:13:08.371 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::LeaderWatcher: 0x55e0369f7b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.372 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79b84a640 -1 rbd::mirror::MirrorStatusUpdater 0x55e036714480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.372 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79b84a640 -1 rbd::mirror::MirrorStatusUpdater 0x55e0367145a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.372 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc79b84a640 -1 rbd::mirror::MirrorStatusUpdater 0x55e036714360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a8864640 -1 rbd::mirror::NamespaceReplayer: 0x55e0355d4e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a8864640 -1 rbd::mirror::NamespaceReplayer: 0x55e0355d4700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a8864640 -1 rbd::mirror::NamespaceReplayer: 0x55e036716000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a2057640 -1 rbd::mirror::MirrorStatusUpdater 0x55e036714120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a2057640 -1 rbd::mirror::MirrorStatusUpdater 0x55e036714000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a2057640 -1 rbd::mirror::MirrorStatusUpdater 0x55e036714240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::NamespaceReplayer: 0x55e0355d4e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::NamespaceReplayer: 0x55e0355d4700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.361+0000 7fc7a9065640 -1 rbd::mirror::NamespaceReplayer: 0x55e036716000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.365+0000 7fc7a9065640 -1 rbd::mirror::LeaderWatcher: 0x55e0372aa300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.365+0000 7fc7a9065640 -1 rbd::mirror::LeaderWatcher: 0x55e0372aa300 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-20T22:13:08.373 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.365+0000 7fc7a8864640 -1 rbd::mirror::LeaderWatcher: 0x55e0372aa300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.383 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2026-02-20T22:13:08.384 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T22:13:08.400 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2026-02-20T22:13:08.400 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:08.430 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f685098a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 53580) UID: 1000 2026-02-20T22:13:08.432 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::InstanceWatcher: 0x55af4e223500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6561 object: (2) No such file or directory 2026-02-20T22:13:08.432 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::InstanceWatcher: 0x55af4da71500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6561 object: (2) No such file or directory 2026-02-20T22:13:08.432 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::InstanceWatcher: 0x55af4e223180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6561 object: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::InstanceWatcher: 0x55af4dcb2700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6564 object: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::InstanceWatcher: 0x55af4da71880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6564 object: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684917b640 -1 rbd::mirror::InstanceWatcher: 0x55af4e223500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684917b640 -1 rbd::mirror::InstanceWatcher: 0x55af4da71500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684997c640 -1 rbd::mirror::InstanceWatcher: 0x55af4e223180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f683c161640 -1 rbd::mirror::InstanceWatcher: 0x55af4dcb2700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f683c161640 -1 rbd::mirror::InstanceWatcher: 0x55af4da71880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684296e640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4da7e7e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684296e640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4da7e360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684296e640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4da7e480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::NamespaceReplayer: 0x55af4c725180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.433 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6835153640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4da7e900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6835153640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4da7ec60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::NamespaceReplayer: 0x55af4c724fc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::NamespaceReplayer: 0x55af4c724000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::NamespaceReplayer: 0x55af4dc0a700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::NamespaceReplayer: 0x55af4dc0a8c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684917b640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4da7e000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684f988640 -1 rbd::mirror::NamespaceReplayer: 0x55af4c725180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684917b640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4da7e120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684917b640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4da7e240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.434 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684f988640 -1 rbd::mirror::NamespaceReplayer: 0x55af4c724fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.435 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684f988640 -1 rbd::mirror::NamespaceReplayer: 0x55af4c724000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.435 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f683c161640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4dd2c900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.435 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f683c161640 -1 rbd::mirror::MirrorStatusUpdater 0x55af4dd2ca20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.435 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684f988640 -1 rbd::mirror::NamespaceReplayer: 0x55af4dc0a700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.435 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f684f988640 -1 rbd::mirror::NamespaceReplayer: 0x55af4dc0a8c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.435 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.425+0000 7f6850189640 -1 rbd::mirror::LeaderWatcher: 0x55af4dd5bb00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.435 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.429+0000 7f6850189640 -1 rbd::mirror::LeaderWatcher: 0x55af4d838300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.446 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2026-02-20T22:13:08.446 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T22:13:08.467 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2026-02-20T22:13:08.467 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:08.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd532750640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 53051) UID: 1000 2026-02-20T22:13:08.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd531f4f640 -1 rbd::mirror::InstanceWatcher: 0x562d431a3500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6509 object: (2) No such file or directory 2026-02-20T22:13:08.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd531f4f640 -1 rbd::mirror::InstanceWatcher: 0x562d42de2000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6509 object: (2) No such file or directory 2026-02-20T22:13:08.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd53174e640 -1 rbd::mirror::InstanceWatcher: 0x562d42710000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6506 object: (2) No such file or directory 2026-02-20T22:13:08.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd53174e640 -1 rbd::mirror::InstanceWatcher: 0x562d431a3180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6506 object: (2) No such file or directory 2026-02-20T22:13:08.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd53174e640 -1 rbd::mirror::InstanceWatcher: 0x562d42710380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.6506 object: (2) No such file or directory 2026-02-20T22:13:08.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd51d726640 -1 rbd::mirror::InstanceWatcher: 0x562d431a3500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd51d726640 -1 rbd::mirror::InstanceWatcher: 0x562d42de2000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd52b742640 -1 rbd::mirror::InstanceWatcher: 0x562d42710000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.497 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd52b742640 -1 rbd::mirror::InstanceWatcher: 0x562d431a3180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd52b742640 -1 rbd::mirror::InstanceWatcher: 0x562d42710380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd51771a640 -1 rbd::mirror::MirrorStatusUpdater 0x562d42a6f0e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd51771a640 -1 rbd::mirror::MirrorStatusUpdater 0x562d42a6ed80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd53174e640 -1 rbd::mirror::NamespaceReplayer: 0x562d427e5500 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd53174e640 -1 rbd::mirror::NamespaceReplayer: 0x562d427e56c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd524734640 -1 rbd::mirror::MirrorStatusUpdater 0x562d427e2360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd531f4f640 -1 rbd::mirror::NamespaceReplayer: 0x562d416ae700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd524f35640 -1 rbd::mirror::MirrorStatusUpdater 0x562d427e25a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd524f35640 -1 rbd::mirror::MirrorStatusUpdater 0x562d427e2480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd53174e640 -1 rbd::mirror::NamespaceReplayer: 0x562d416aee00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd53174e640 -1 rbd::mirror::NamespaceReplayer: 0x562d427e4000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.498 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd51df27640 -1 rbd::mirror::MirrorStatusUpdater 0x562d42a6ea20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd51df27640 -1 rbd::mirror::MirrorStatusUpdater 0x562d42a6eb40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd531f4f640 -1 rbd::mirror::NamespaceReplayer: 0x562d427e5500 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd53174e640 -1 rbd::mirror::NamespaceReplayer: 0x562d427e56c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd52af41640 -1 rbd::mirror::MirrorStatusUpdater 0x562d427e2000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd52af41640 -1 rbd::mirror::MirrorStatusUpdater 0x562d427e2120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd52af41640 -1 rbd::mirror::MirrorStatusUpdater 0x562d427e2240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd531f4f640 -1 rbd::mirror::NamespaceReplayer: 0x562d416ae700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd531f4f640 -1 rbd::mirror::NamespaceReplayer: 0x562d416aee00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd531f4f640 -1 rbd::mirror::NamespaceReplayer: 0x562d427e4000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.499 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd531f4f640 -1 rbd::mirror::LeaderWatcher: 0x562d42ad9b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.500 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.489+0000 7fd531f4f640 -1 rbd::mirror::LeaderWatcher: 0x562d434cc300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.511 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2026-02-20T22:13:08.511 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T22:13:08.529 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2026-02-20T22:13:08.529 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:08.554 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531de9c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 42210) UID: 1000 2026-02-20T22:13:08.557 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::InstanceWatcher: 0x5617463f2000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5826 object: (2) No such file or directory 2026-02-20T22:13:08.557 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::InstanceWatcher: 0x561746e71180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5826 object: (2) No such file or directory 2026-02-20T22:13:08.557 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::InstanceWatcher: 0x5617463f2380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5826 object: (2) No such file or directory 2026-02-20T22:13:08.557 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::InstanceWatcher: 0x561746e71500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5828 object: (2) No such file or directory 2026-02-20T22:13:08.557 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::InstanceWatcher: 0x5617463f3c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5828 object: (2) No such file or directory 2026-02-20T22:13:08.557 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531668d640 -1 rbd::mirror::InstanceWatcher: 0x5617463f2000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.557 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531668d640 -1 rbd::mirror::InstanceWatcher: 0x5617463f2380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.557 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f5309673640 -1 rbd::mirror::InstanceWatcher: 0x561746e71500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f5309673640 -1 rbd::mirror::InstanceWatcher: 0x5617463f3c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f5316e8e640 -1 rbd::mirror::InstanceWatcher: 0x561746e71180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531668d640 -1 rbd::mirror::MirrorStatusUpdater 0x5617464ac240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::NamespaceReplayer: 0x561746488380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f530fe80640 -1 rbd::mirror::MirrorStatusUpdater 0x5617464ac5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::NamespaceReplayer: 0x561746488000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f5310681640 -1 rbd::mirror::MirrorStatusUpdater 0x5617464ac360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531ce9a640 -1 rbd::mirror::NamespaceReplayer: 0x5617464881c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f5302e66640 -1 rbd::mirror::MirrorStatusUpdater 0x56174676afc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f5302e66640 -1 rbd::mirror::MirrorStatusUpdater 0x56174676ac60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::NamespaceReplayer: 0x561746489180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f5316e8e640 -1 rbd::mirror::MirrorStatusUpdater 0x5617464ac000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.558 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::NamespaceReplayer: 0x561746489340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.559 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::NamespaceReplayer: 0x561746488000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.559 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531668d640 -1 rbd::mirror::MirrorStatusUpdater 0x5617464ac120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.559 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531d69b640 -1 rbd::mirror::NamespaceReplayer: 0x5617464881c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.559 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f5308e72640 -1 rbd::mirror::MirrorStatusUpdater 0x56174676a900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.559 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f5308e72640 -1 rbd::mirror::MirrorStatusUpdater 0x56174676aa20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.559 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531ce9a640 -1 rbd::mirror::NamespaceReplayer: 0x561746489180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.559 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531ce9a640 -1 rbd::mirror::NamespaceReplayer: 0x561746489340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.559 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.549+0000 7f531ce9a640 -1 rbd::mirror::LeaderWatcher: 0x561746799b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.560 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.553+0000 7f531ce9a640 -1 rbd::mirror::LeaderWatcher: 0x56174719a300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.572 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2026-02-20T22:13:08.573 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T22:13:08.591 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2026-02-20T22:13:08.591 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:08.614 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740e108640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 19388) UID: 1000 2026-02-20T22:13:08.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f90de640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564a7a266b40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:08.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f90de640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x564a7a142480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T22:13:08.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f98df640 -1 rbd::mirror::PoolWatcher: 0x564a7a0cb680 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f98df640 -1 rbd::mirror::PoolWatcher: 0x564a79d8e000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f30d2640 -1 rbd::mirror::PoolWatcher: 0x564a794a45a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.617 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f28d1640 -1 rbd::mirror::PoolWatcher: 0x564a79d8f2c0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d907640 -1 rbd::mirror::InstanceWatcher: 0x564a794b9500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4272 object: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::InstanceWatcher: 0x564a79aac380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4272 object: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::InstanceWatcher: 0x564a79aac700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4272 object: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::InstanceWatcher: 0x564a79aaca80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4287 object: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::InstanceWatcher: 0x564a79aace00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4287 object: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f74068f9640 -1 rbd::mirror::InstanceWatcher: 0x564a794b9500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f74070fa640 -1 rbd::mirror::InstanceWatcher: 0x564a79aac380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f74070fa640 -1 rbd::mirror::InstanceWatcher: 0x564a79aac700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f98df640 -1 rbd::mirror::InstanceWatcher: 0x564a79aaca80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f74008ed640 -1 rbd::mirror::MirrorStatusUpdater 0x564a79966d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f74008ed640 -1 rbd::mirror::MirrorStatusUpdater 0x564a79967320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.618 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f74008ed640 -1 rbd::mirror::MirrorStatusUpdater 0x564a79967200 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f90de640 -1 rbd::mirror::InstanceWatcher: 0x564a79aace00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1a380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1a540 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1a700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f28d1640 -1 rbd::mirror::MirrorStatusUpdater 0x564a79967b00 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f74068f9640 -1 rbd::mirror::MirrorStatusUpdater 0x564a799667e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f28d1640 -1 rbd::mirror::MirrorStatusUpdater 0x564a799677a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1a8c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1a380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1aa80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f74070fa640 -1 rbd::mirror::MirrorStatusUpdater 0x564a79966900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.619 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f74070fa640 -1 rbd::mirror::MirrorStatusUpdater 0x564a79966a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1a540 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1a700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f90de640 -1 rbd::mirror::MirrorStatusUpdater 0x564a79966ea0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f73f90de640 -1 rbd::mirror::MirrorStatusUpdater 0x564a79966fc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1a8c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::NamespaceReplayer: 0x564a79b1aa80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::LeaderWatcher: 0x564a799cd500 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-20T22:13:08.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d106640 -1 rbd::mirror::LeaderWatcher: 0x564a799cd500 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-20T22:13:08.620 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.609+0000 7f740d907640 -1 rbd::mirror::LeaderWatcher: 0x564a799cd500 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.621 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial107.stderr:2026-02-20T22:13:08.613+0000 7f740d907640 -1 rbd::mirror::LeaderWatcher: 0x564a7a04b800 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.636 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2026-02-20T22:13:08.637 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T22:13:08.654 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2026-02-20T22:13:08.655 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:08.678 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0e252640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 46317) UID: 1000 2026-02-20T22:13:08.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0d250640 -1 rbd::mirror::InstanceWatcher: 0x556c79b73180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5584 object: (2) No such file or directory 2026-02-20T22:13:08.681 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0da51640 -1 rbd::mirror::InstanceWatcher: 0x556c7920a000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5584 object: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0da51640 -1 rbd::mirror::InstanceWatcher: 0x556c7920a380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5584 object: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d06a43640 -1 rbd::mirror::InstanceWatcher: 0x556c79b73180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d07244640 -1 rbd::mirror::InstanceWatcher: 0x556c7920a000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d06a43640 -1 rbd::mirror::InstanceWatcher: 0x556c7920a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d00236640 -1 rbd::mirror::MirrorStatusUpdater 0x556c791c05a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0da51640 -1 rbd::mirror::NamespaceReplayer: 0x556c791be000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d00a37640 -1 rbd::mirror::MirrorStatusUpdater 0x556c791c0480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0d250640 -1 rbd::mirror::NamespaceReplayer: 0x556c791be1c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d00236640 -1 rbd::mirror::MirrorStatusUpdater 0x556c791c0360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.682 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0da51640 -1 rbd::mirror::NamespaceReplayer: 0x556c791be380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d07244640 -1 rbd::mirror::MirrorStatusUpdater 0x556c791c0000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0d250640 -1 rbd::mirror::NamespaceReplayer: 0x556c791be000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d06a43640 -1 rbd::mirror::MirrorStatusUpdater 0x556c791c0120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0d250640 -1 rbd::mirror::NamespaceReplayer: 0x556c791be1c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d07244640 -1 rbd::mirror::MirrorStatusUpdater 0x556c791c0240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0d250640 -1 rbd::mirror::NamespaceReplayer: 0x556c791be380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0da51640 -1 rbd::mirror::InstanceWatcher: 0x556c797cc000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5586 object: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0da51640 -1 rbd::mirror::InstanceWatcher: 0x556c797cca80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5586 object: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1cf9228640 -1 rbd::mirror::InstanceWatcher: 0x556c797cc000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1cf9a29640 -1 rbd::mirror::InstanceWatcher: 0x556c797cca80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1cf321c640 -1 rbd::mirror::MirrorStatusUpdater 0x556c791c06c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0d250640 -1 rbd::mirror::NamespaceReplayer: 0x556c791befc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.683 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1cf9228640 -1 rbd::mirror::MirrorStatusUpdater 0x556c79428d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.684 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0da51640 -1 rbd::mirror::NamespaceReplayer: 0x556c791befc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.684 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.673+0000 7f1d0da51640 -1 rbd::mirror::LeaderWatcher: 0x556c79489b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.684 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial107.stderr:2026-02-20T22:13:08.677+0000 7f1d0da51640 -1 rbd::mirror::LeaderWatcher: 0x556c79e9c300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.696 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2026-02-20T22:13:08.696 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T22:13:08.717 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2026-02-20T22:13:08.717 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:08.738 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3377171640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 48301) UID: 1000 2026-02-20T22:13:08.742 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::InstanceWatcher: 0x55b1d82a0380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5602 object: (2) No such file or directory 2026-02-20T22:13:08.742 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f337616f640 -1 rbd::mirror::InstanceWatcher: 0x55b1d7eb5c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5602 object: (2) No such file or directory 2026-02-20T22:13:08.742 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3362948640 -1 rbd::mirror::InstanceWatcher: 0x55b1d82a0380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.742 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3362147640 -1 rbd::mirror::InstanceWatcher: 0x55b1d7eb5c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.742 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f335c13b640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d8096c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.742 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f335c13b640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d8096fc0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.742 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f337616f640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d7f4afc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f337616f640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d7f4b180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3362948640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d8096900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d7f4afc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3362147640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d8096a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d7f4b180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f337616f640 -1 rbd::mirror::LeaderWatcher: 0x55b1d80c3b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::InstanceWatcher: 0x55b1d87ad180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5599 object: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::InstanceWatcher: 0x55b1d87adc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5599 object: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::InstanceWatcher: 0x55b1d7eb5880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5599 object: (2) No such file or directory 2026-02-20T22:13:08.743 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f336f962640 -1 rbd::mirror::InstanceWatcher: 0x55b1d87ad180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f336f962640 -1 rbd::mirror::InstanceWatcher: 0x55b1d87adc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f336f962640 -1 rbd::mirror::InstanceWatcher: 0x55b1d7eb5880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3369155640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d7d927e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3369155640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d7d926c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3369155640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d7d925a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d6ccc000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d6cccfc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d6ccce00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3370163640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d7d92240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f337616f640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d6ccc000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.744 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3370163640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d7d92480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.745 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3370163640 -1 rbd::mirror::MirrorStatusUpdater 0x55b1d7d92360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.745 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f337616f640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d6ccce00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.745 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.733+0000 7f3376970640 -1 rbd::mirror::NamespaceReplayer: 0x55b1d6cccfc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.745 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial107.stderr:2026-02-20T22:13:08.737+0000 7f3376970640 -1 rbd::mirror::LeaderWatcher: 0x55b1d8ae8300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.755 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2026-02-20T22:13:08.756 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T22:13:08.776 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2026-02-20T22:13:08.776 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:08.802 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915bb21640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 52709) UID: 1000 2026-02-20T22:13:08.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::InstanceWatcher: 0x558d3bb99500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5950 object: (2) No such file or directory 2026-02-20T22:13:08.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915ab1f640 -1 rbd::mirror::InstanceWatcher: 0x558d3b29f500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5952 object: (2) No such file or directory 2026-02-20T22:13:08.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::InstanceWatcher: 0x558d3b29e380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5950 object: (2) No such file or directory 2026-02-20T22:13:08.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915ab1f640 -1 rbd::mirror::InstanceWatcher: 0x558d3b29fc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5952 object: (2) No such file or directory 2026-02-20T22:13:08.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::InstanceWatcher: 0x558d3bb99180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5950 object: (2) No such file or directory 2026-02-20T22:13:08.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f91472f8640 -1 rbd::mirror::InstanceWatcher: 0x558d3b29f500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f9154b13640 -1 rbd::mirror::InstanceWatcher: 0x558d3bb99500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.806 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f9154312640 -1 rbd::mirror::InstanceWatcher: 0x558d3b29e380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f914db05640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b1e05a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::NamespaceReplayer: 0x558d3a0bce00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f914e306640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b1e0360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f9154b13640 -1 rbd::mirror::InstanceWatcher: 0x558d3bb99180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f91402ea640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b4490e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::NamespaceReplayer: 0x558d3b1e21c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::NamespaceReplayer: 0x558d3b1e2380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f914db05640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b1e0480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::NamespaceReplayer: 0x558d3b1e2000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.807 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f9154312640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b1e0000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915ab1f640 -1 rbd::mirror::NamespaceReplayer: 0x558d3a0bce00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f9154b13640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b1e0240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::NamespaceReplayer: 0x558d3b1e21c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f9154312640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b1e0120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915ab1f640 -1 rbd::mirror::NamespaceReplayer: 0x558d3b1e2000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f9146af7640 -1 rbd::mirror::InstanceWatcher: 0x558d3b29fc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f9146af7640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b448a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::NamespaceReplayer: 0x558d3b1e2380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f91402ea640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b448d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915ab1f640 -1 rbd::mirror::NamespaceReplayer: 0x558d3b1e2c40 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.808 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f91472f8640 -1 rbd::mirror::MirrorStatusUpdater 0x558d3b448b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T22:13:08.809 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.797+0000 7f915b320640 -1 rbd::mirror::NamespaceReplayer: 0x558d3b1e2c40 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T22:13:08.809 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.801+0000 7f915ab1f640 -1 rbd::mirror::LeaderWatcher: 0x558d3b4adb00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.809 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial107.stderr:2026-02-20T22:13:08.801+0000 7f915ab1f640 -1 rbd::mirror::LeaderWatcher: 0x558d3bed0300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T22:13:08.821 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2026-02-20T22:13:08.821 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-20T22:13:08.841 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-20T22:13:08.842 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T22:13:09.011 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T22:13:09.011 INFO:teuthology.orchestra.run.trial107.stderr:dumped all 2026-02-20T22:13:09.023 INFO:teuthology.orchestra.run.trial107.stdout:{"pg_ready":true,"pg_map":{"version":1567,"stamp":"2026-02-20T22:13:08.608869+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4576352,"kb_used_data":17664,"kb_used_omap":1925,"kb_used_meta":4556154,"kb_avail":278539168,"statfs":{"total":289910292480,"available":285224108032,"internally_reserved":0,"allocated":18087936,"data_stored":10137204,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":1971213,"internal_metadata":4665502707},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536871032,"num_objects":-174,"num_object_clones":0,"num_object_copies":-348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-174,"num_whiteouts":0,"num_read":-136375,"num_read_kb":-39848963,"num_write":-1449189,"num_write_kb":-540169603,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-1,"num_objects_omap":-18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001281"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.786156+0000","last_change":"2026-02-20T21:37:43.629317+0000","last_active":"2026-02-20T22:13:05.786156+0000","last_peered":"2026-02-20T22:13:05.786156+0000","last_clean":"2026-02-20T22:13:05.786156+0000","last_became_active":"2026-02-20T21:37:41.626252+0000","last_became_peered":"2026-02-20T21:37:41.626252+0000","last_unstale":"2026-02-20T22:13:05.786156+0000","last_undegraded":"2026-02-20T22:13:05.786156+0000","last_fullsized":"2026-02-20T22:13:05.786156+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:57:26.704591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000878194,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.791963+0000","last_change":"2026-02-20T21:37:43.628857+0000","last_active":"2026-02-20T22:13:05.791963+0000","last_peered":"2026-02-20T22:13:05.791963+0000","last_clean":"2026-02-20T22:13:05.791963+0000","last_became_active":"2026-02-20T21:37:41.626370+0000","last_became_peered":"2026-02-20T21:37:41.626370+0000","last_unstale":"2026-02-20T22:13:05.791963+0000","last_undegraded":"2026-02-20T22:13:05.791963+0000","last_fullsized":"2026-02-20T22:13:05.791963+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:39:08.125334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050412799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.789961+0000","last_change":"2026-02-20T21:37:43.629072+0000","last_active":"2026-02-20T22:13:05.789961+0000","last_peered":"2026-02-20T22:13:05.789961+0000","last_clean":"2026-02-20T22:13:05.789961+0000","last_became_active":"2026-02-20T21:37:41.626503+0000","last_became_peered":"2026-02-20T21:37:41.626503+0000","last_unstale":"2026-02-20T22:13:05.789961+0000","last_undegraded":"2026-02-20T22:13:05.789961+0000","last_fullsized":"2026-02-20T22:13:05.789961+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:55:04.979567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000697728,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1015,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.784329+0000","last_change":"2026-02-20T21:37:43.628823+0000","last_active":"2026-02-20T22:13:05.784329+0000","last_peered":"2026-02-20T22:13:05.784329+0000","last_clean":"2026-02-20T22:13:05.784329+0000","last_became_active":"2026-02-20T21:37:41.626044+0000","last_became_peered":"2026-02-20T21:37:41.626044+0000","last_unstale":"2026-02-20T22:13:05.784329+0000","last_undegraded":"2026-02-20T22:13:05.784329+0000","last_fullsized":"2026-02-20T22:13:05.784329+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:43:21.886202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052464600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":1013,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-20T22:12:59.776799+0000","last_change":"2026-02-20T21:37:43.627134+0000","last_active":"2026-02-20T22:12:59.776799+0000","last_peered":"2026-02-20T22:12:59.776799+0000","last_clean":"2026-02-20T22:12:59.776799+0000","last_became_active":"2026-02-20T21:37:41.626685+0000","last_became_peered":"2026-02-20T21:37:41.626685+0000","last_unstale":"2026-02-20T22:12:59.776799+0000","last_undegraded":"2026-02-20T22:12:59.776799+0000","last_fullsized":"2026-02-20T22:12:59.776799+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:19:24.951160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066038900000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1012,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-20T22:12:59.781120+0000","last_change":"2026-02-20T21:37:43.653844+0000","last_active":"2026-02-20T22:12:59.781120+0000","last_peered":"2026-02-20T22:12:59.781120+0000","last_clean":"2026-02-20T22:12:59.781120+0000","last_became_active":"2026-02-20T21:37:41.626554+0000","last_became_peered":"2026-02-20T21:37:41.626554+0000","last_unstale":"2026-02-20T22:12:59.781120+0000","last_undegraded":"2026-02-20T22:12:59.781120+0000","last_fullsized":"2026-02-20T22:12:59.781120+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:23:54.818688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021567099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1013,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-20T22:12:59.781055+0000","last_change":"2026-02-20T21:37:43.653839+0000","last_active":"2026-02-20T22:12:59.781055+0000","last_peered":"2026-02-20T22:12:59.781055+0000","last_clean":"2026-02-20T22:12:59.781055+0000","last_became_active":"2026-02-20T21:37:41.626692+0000","last_became_peered":"2026-02-20T21:37:41.626692+0000","last_unstale":"2026-02-20T22:12:59.781055+0000","last_undegraded":"2026-02-20T22:12:59.781055+0000","last_fullsized":"2026-02-20T22:12:59.781055+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:10:13.031958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021587200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":1016,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.792266+0000","last_change":"2026-02-20T21:37:43.629180+0000","last_active":"2026-02-20T22:13:05.792266+0000","last_peered":"2026-02-20T22:13:05.792266+0000","last_clean":"2026-02-20T22:13:05.792266+0000","last_became_active":"2026-02-20T21:37:41.626007+0000","last_became_peered":"2026-02-20T21:37:41.626007+0000","last_unstale":"2026-02-20T22:13:05.792266+0000","last_undegraded":"2026-02-20T22:13:05.792266+0000","last_fullsized":"2026-02-20T22:13:05.792266+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:01:20.786792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080498799999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1104,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.790346+0000","last_change":"2026-02-20T21:37:39.618505+0000","last_active":"2026-02-20T22:13:05.790346+0000","last_peered":"2026-02-20T22:13:05.790346+0000","last_clean":"2026-02-20T22:13:05.790346+0000","last_became_active":"2026-02-20T21:37:39.618273+0000","last_became_peered":"2026-02-20T21:37:39.618273+0000","last_unstale":"2026-02-20T22:13:05.790346+0000","last_undegraded":"2026-02-20T22:13:05.790346+0000","last_fullsized":"2026-02-20T22:13:05.790346+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_clean_scrub_stamp":"2026-02-20T21:37:38.611598+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:26:23.564860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738796,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1277624,"kb_used_data":5840,"kb_used_omap":1247,"kb_used_meta":1270240,"kb_avail":93094216,"statfs":{"total":96636764160,"available":95328477184,"internally_reserved":0,"allocated":5980160,"data_stored":2983107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1277707,"internal_metadata":1300726005},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738797,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1841272,"kb_used_data":5904,"kb_used_omap":195,"kb_used_meta":1834876,"kb_avail":92530568,"statfs":{"total":96636764160,"available":94751301632,"internally_reserved":0,"allocated":6045696,"data_stored":3573475,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":200482,"internal_metadata":1878913246},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738796,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1457456,"kb_used_data":5920,"kb_used_omap":481,"kb_used_meta":1451038,"kb_avail":92914384,"statfs":{"total":96636764160,"available":95144329216,"internally_reserved":0,"allocated":6062080,"data_stored":3580622,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":493024,"internal_metadata":1485863456},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:13:09.025 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T22:13:09.173 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T22:13:09.174 INFO:teuthology.orchestra.run.trial107.stderr:dumped all 2026-02-20T22:13:09.184 INFO:teuthology.orchestra.run.trial107.stdout:{"pg_ready":true,"pg_map":{"version":1567,"stamp":"2026-02-20T22:13:08.608869+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4576352,"kb_used_data":17664,"kb_used_omap":1925,"kb_used_meta":4556154,"kb_avail":278539168,"statfs":{"total":289910292480,"available":285224108032,"internally_reserved":0,"allocated":18087936,"data_stored":10137204,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":1971213,"internal_metadata":4665502707},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536871032,"num_objects":-174,"num_object_clones":0,"num_object_copies":-348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-174,"num_whiteouts":0,"num_read":-136375,"num_read_kb":-39848963,"num_write":-1449189,"num_write_kb":-540169603,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-1,"num_objects_omap":-18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001281"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.786156+0000","last_change":"2026-02-20T21:37:43.629317+0000","last_active":"2026-02-20T22:13:05.786156+0000","last_peered":"2026-02-20T22:13:05.786156+0000","last_clean":"2026-02-20T22:13:05.786156+0000","last_became_active":"2026-02-20T21:37:41.626252+0000","last_became_peered":"2026-02-20T21:37:41.626252+0000","last_unstale":"2026-02-20T22:13:05.786156+0000","last_undegraded":"2026-02-20T22:13:05.786156+0000","last_fullsized":"2026-02-20T22:13:05.786156+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:57:26.704591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000878194,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.791963+0000","last_change":"2026-02-20T21:37:43.628857+0000","last_active":"2026-02-20T22:13:05.791963+0000","last_peered":"2026-02-20T22:13:05.791963+0000","last_clean":"2026-02-20T22:13:05.791963+0000","last_became_active":"2026-02-20T21:37:41.626370+0000","last_became_peered":"2026-02-20T21:37:41.626370+0000","last_unstale":"2026-02-20T22:13:05.791963+0000","last_undegraded":"2026-02-20T22:13:05.791963+0000","last_fullsized":"2026-02-20T22:13:05.791963+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:39:08.125334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050412799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.789961+0000","last_change":"2026-02-20T21:37:43.629072+0000","last_active":"2026-02-20T22:13:05.789961+0000","last_peered":"2026-02-20T22:13:05.789961+0000","last_clean":"2026-02-20T22:13:05.789961+0000","last_became_active":"2026-02-20T21:37:41.626503+0000","last_became_peered":"2026-02-20T21:37:41.626503+0000","last_unstale":"2026-02-20T22:13:05.789961+0000","last_undegraded":"2026-02-20T22:13:05.789961+0000","last_fullsized":"2026-02-20T22:13:05.789961+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:55:04.979567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000697728,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1015,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.784329+0000","last_change":"2026-02-20T21:37:43.628823+0000","last_active":"2026-02-20T22:13:05.784329+0000","last_peered":"2026-02-20T22:13:05.784329+0000","last_clean":"2026-02-20T22:13:05.784329+0000","last_became_active":"2026-02-20T21:37:41.626044+0000","last_became_peered":"2026-02-20T21:37:41.626044+0000","last_unstale":"2026-02-20T22:13:05.784329+0000","last_undegraded":"2026-02-20T22:13:05.784329+0000","last_fullsized":"2026-02-20T22:13:05.784329+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:43:21.886202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052464600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":1013,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-20T22:12:59.776799+0000","last_change":"2026-02-20T21:37:43.627134+0000","last_active":"2026-02-20T22:12:59.776799+0000","last_peered":"2026-02-20T22:12:59.776799+0000","last_clean":"2026-02-20T22:12:59.776799+0000","last_became_active":"2026-02-20T21:37:41.626685+0000","last_became_peered":"2026-02-20T21:37:41.626685+0000","last_unstale":"2026-02-20T22:12:59.776799+0000","last_undegraded":"2026-02-20T22:12:59.776799+0000","last_fullsized":"2026-02-20T22:12:59.776799+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:19:24.951160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066038900000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1012,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-20T22:12:59.781120+0000","last_change":"2026-02-20T21:37:43.653844+0000","last_active":"2026-02-20T22:12:59.781120+0000","last_peered":"2026-02-20T22:12:59.781120+0000","last_clean":"2026-02-20T22:12:59.781120+0000","last_became_active":"2026-02-20T21:37:41.626554+0000","last_became_peered":"2026-02-20T21:37:41.626554+0000","last_unstale":"2026-02-20T22:12:59.781120+0000","last_undegraded":"2026-02-20T22:12:59.781120+0000","last_fullsized":"2026-02-20T22:12:59.781120+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:23:54.818688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021567099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1013,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-20T22:12:59.781055+0000","last_change":"2026-02-20T21:37:43.653839+0000","last_active":"2026-02-20T22:12:59.781055+0000","last_peered":"2026-02-20T22:12:59.781055+0000","last_clean":"2026-02-20T22:12:59.781055+0000","last_became_active":"2026-02-20T21:37:41.626692+0000","last_became_peered":"2026-02-20T21:37:41.626692+0000","last_unstale":"2026-02-20T22:12:59.781055+0000","last_undegraded":"2026-02-20T22:12:59.781055+0000","last_fullsized":"2026-02-20T22:12:59.781055+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:10:13.031958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021587200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":1016,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.792266+0000","last_change":"2026-02-20T21:37:43.629180+0000","last_active":"2026-02-20T22:13:05.792266+0000","last_peered":"2026-02-20T22:13:05.792266+0000","last_clean":"2026-02-20T22:13:05.792266+0000","last_became_active":"2026-02-20T21:37:41.626007+0000","last_became_peered":"2026-02-20T21:37:41.626007+0000","last_unstale":"2026-02-20T22:13:05.792266+0000","last_undegraded":"2026-02-20T22:13:05.792266+0000","last_fullsized":"2026-02-20T22:13:05.792266+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:01:20.786792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080498799999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1104,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.790346+0000","last_change":"2026-02-20T21:37:39.618505+0000","last_active":"2026-02-20T22:13:05.790346+0000","last_peered":"2026-02-20T22:13:05.790346+0000","last_clean":"2026-02-20T22:13:05.790346+0000","last_became_active":"2026-02-20T21:37:39.618273+0000","last_became_peered":"2026-02-20T21:37:39.618273+0000","last_unstale":"2026-02-20T22:13:05.790346+0000","last_undegraded":"2026-02-20T22:13:05.790346+0000","last_fullsized":"2026-02-20T22:13:05.790346+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_clean_scrub_stamp":"2026-02-20T21:37:38.611598+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:26:23.564860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738796,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1277624,"kb_used_data":5840,"kb_used_omap":1247,"kb_used_meta":1270240,"kb_avail":93094216,"statfs":{"total":96636764160,"available":95328477184,"internally_reserved":0,"allocated":5980160,"data_stored":2983107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1277707,"internal_metadata":1300726005},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738797,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1841272,"kb_used_data":5904,"kb_used_omap":195,"kb_used_meta":1834876,"kb_avail":92530568,"statfs":{"total":96636764160,"available":94751301632,"internally_reserved":0,"allocated":6045696,"data_stored":3573475,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":200482,"internal_metadata":1878913246},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738796,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1457456,"kb_used_data":5920,"kb_used_omap":481,"kb_used_meta":1451038,"kb_avail":92914384,"statfs":{"total":96636764160,"available":95144329216,"internally_reserved":0,"allocated":6062080,"data_stored":3580622,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":493024,"internal_metadata":1485863456},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:13:09.185 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-20T22:13:09.186 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T22:13:09.335 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T22:13:09.336 INFO:teuthology.orchestra.run.trial107.stderr:dumped all 2026-02-20T22:13:09.346 INFO:teuthology.orchestra.run.trial107.stdout:{"pg_ready":true,"pg_map":{"version":1567,"stamp":"2026-02-20T22:13:08.608869+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4576352,"kb_used_data":17664,"kb_used_omap":1925,"kb_used_meta":4556154,"kb_avail":278539168,"statfs":{"total":289910292480,"available":285224108032,"internally_reserved":0,"allocated":18087936,"data_stored":10137204,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":1971213,"internal_metadata":4665502707},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536871032,"num_objects":-174,"num_object_clones":0,"num_object_copies":-348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-174,"num_whiteouts":0,"num_read":-136375,"num_read_kb":-39848963,"num_write":-1449189,"num_write_kb":-540169603,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-1,"num_objects_omap":-18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001281"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.786156+0000","last_change":"2026-02-20T21:37:43.629317+0000","last_active":"2026-02-20T22:13:05.786156+0000","last_peered":"2026-02-20T22:13:05.786156+0000","last_clean":"2026-02-20T22:13:05.786156+0000","last_became_active":"2026-02-20T21:37:41.626252+0000","last_became_peered":"2026-02-20T21:37:41.626252+0000","last_unstale":"2026-02-20T22:13:05.786156+0000","last_undegraded":"2026-02-20T22:13:05.786156+0000","last_fullsized":"2026-02-20T22:13:05.786156+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:57:26.704591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000878194,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.791963+0000","last_change":"2026-02-20T21:37:43.628857+0000","last_active":"2026-02-20T22:13:05.791963+0000","last_peered":"2026-02-20T22:13:05.791963+0000","last_clean":"2026-02-20T22:13:05.791963+0000","last_became_active":"2026-02-20T21:37:41.626370+0000","last_became_peered":"2026-02-20T21:37:41.626370+0000","last_unstale":"2026-02-20T22:13:05.791963+0000","last_undegraded":"2026-02-20T22:13:05.791963+0000","last_fullsized":"2026-02-20T22:13:05.791963+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:39:08.125334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050412799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.789961+0000","last_change":"2026-02-20T21:37:43.629072+0000","last_active":"2026-02-20T22:13:05.789961+0000","last_peered":"2026-02-20T22:13:05.789961+0000","last_clean":"2026-02-20T22:13:05.789961+0000","last_became_active":"2026-02-20T21:37:41.626503+0000","last_became_peered":"2026-02-20T21:37:41.626503+0000","last_unstale":"2026-02-20T22:13:05.789961+0000","last_undegraded":"2026-02-20T22:13:05.789961+0000","last_fullsized":"2026-02-20T22:13:05.789961+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:55:04.979567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000697728,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1015,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.784329+0000","last_change":"2026-02-20T21:37:43.628823+0000","last_active":"2026-02-20T22:13:05.784329+0000","last_peered":"2026-02-20T22:13:05.784329+0000","last_clean":"2026-02-20T22:13:05.784329+0000","last_became_active":"2026-02-20T21:37:41.626044+0000","last_became_peered":"2026-02-20T21:37:41.626044+0000","last_unstale":"2026-02-20T22:13:05.784329+0000","last_undegraded":"2026-02-20T22:13:05.784329+0000","last_fullsized":"2026-02-20T22:13:05.784329+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:43:21.886202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052464600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":1013,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-20T22:12:59.776799+0000","last_change":"2026-02-20T21:37:43.627134+0000","last_active":"2026-02-20T22:12:59.776799+0000","last_peered":"2026-02-20T22:12:59.776799+0000","last_clean":"2026-02-20T22:12:59.776799+0000","last_became_active":"2026-02-20T21:37:41.626685+0000","last_became_peered":"2026-02-20T21:37:41.626685+0000","last_unstale":"2026-02-20T22:12:59.776799+0000","last_undegraded":"2026-02-20T22:12:59.776799+0000","last_fullsized":"2026-02-20T22:12:59.776799+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:19:24.951160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066038900000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1012,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-20T22:12:59.781120+0000","last_change":"2026-02-20T21:37:43.653844+0000","last_active":"2026-02-20T22:12:59.781120+0000","last_peered":"2026-02-20T22:12:59.781120+0000","last_clean":"2026-02-20T22:12:59.781120+0000","last_became_active":"2026-02-20T21:37:41.626554+0000","last_became_peered":"2026-02-20T21:37:41.626554+0000","last_unstale":"2026-02-20T22:12:59.781120+0000","last_undegraded":"2026-02-20T22:12:59.781120+0000","last_fullsized":"2026-02-20T22:12:59.781120+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:23:54.818688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021567099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1013,"reported_epoch":502,"state":"active+clean","last_fresh":"2026-02-20T22:12:59.781055+0000","last_change":"2026-02-20T21:37:43.653839+0000","last_active":"2026-02-20T22:12:59.781055+0000","last_peered":"2026-02-20T22:12:59.781055+0000","last_clean":"2026-02-20T22:12:59.781055+0000","last_became_active":"2026-02-20T21:37:41.626692+0000","last_became_peered":"2026-02-20T21:37:41.626692+0000","last_unstale":"2026-02-20T22:12:59.781055+0000","last_undegraded":"2026-02-20T22:12:59.781055+0000","last_fullsized":"2026-02-20T22:12:59.781055+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:10:13.031958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021587200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":1016,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.792266+0000","last_change":"2026-02-20T21:37:43.629180+0000","last_active":"2026-02-20T22:13:05.792266+0000","last_peered":"2026-02-20T22:13:05.792266+0000","last_clean":"2026-02-20T22:13:05.792266+0000","last_became_active":"2026-02-20T21:37:41.626007+0000","last_became_peered":"2026-02-20T21:37:41.626007+0000","last_unstale":"2026-02-20T22:13:05.792266+0000","last_undegraded":"2026-02-20T22:13:05.792266+0000","last_fullsized":"2026-02-20T22:13:05.792266+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:01:20.786792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080498799999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1104,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.790346+0000","last_change":"2026-02-20T21:37:39.618505+0000","last_active":"2026-02-20T22:13:05.790346+0000","last_peered":"2026-02-20T22:13:05.790346+0000","last_clean":"2026-02-20T22:13:05.790346+0000","last_became_active":"2026-02-20T21:37:39.618273+0000","last_became_peered":"2026-02-20T21:37:39.618273+0000","last_unstale":"2026-02-20T22:13:05.790346+0000","last_undegraded":"2026-02-20T22:13:05.790346+0000","last_fullsized":"2026-02-20T22:13:05.790346+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_clean_scrub_stamp":"2026-02-20T21:37:38.611598+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:26:23.564860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738796,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1277624,"kb_used_data":5840,"kb_used_omap":1247,"kb_used_meta":1270240,"kb_avail":93094216,"statfs":{"total":96636764160,"available":95328477184,"internally_reserved":0,"allocated":5980160,"data_stored":2983107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1277707,"internal_metadata":1300726005},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738797,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1841272,"kb_used_data":5904,"kb_used_omap":195,"kb_used_meta":1834876,"kb_avail":92530568,"statfs":{"total":96636764160,"available":94751301632,"internally_reserved":0,"allocated":6045696,"data_stored":3573475,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":200482,"internal_metadata":1878913246},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738796,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1457456,"kb_used_data":5920,"kb_used_omap":481,"kb_used_meta":1451038,"kb_avail":92914384,"statfs":{"total":96636764160,"available":95144329216,"internally_reserved":0,"allocated":6062080,"data_stored":3580622,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":493024,"internal_metadata":1485863456},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:13:09.348 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T22:13:09.506 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T22:13:09.506 INFO:teuthology.orchestra.run.trial107.stdout:{"epoch":504,"fsid":"e67c6e85-7576-4986-9d91-ad2adc537194","created":"2026-02-20T21:37:32.598295+0000","modified":"2026-02-20T22:13:07.785427+0000","last_up_change":"2026-02-20T21:37:37.608773+0000","last_in_change":"2026-02-20T21:37:33.219748+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:37:38.286097+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:37:40.560120+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"418d8210-8c01-45cb-abd1-b38a88311d87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6800","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6801","nonce":4201493740}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6802","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6803","nonce":4201493740}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6806","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6807","nonce":4201493740}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6804","nonce":4201493740},{"type":"v1","addr":"10.20.193.107:6805","nonce":4201493740}]},"public_addr":"10.20.193.107:6801/4201493740","cluster_addr":"10.20.193.107:6803/4201493740","heartbeat_back_addr":"10.20.193.107:6807/4201493740","heartbeat_front_addr":"10.20.193.107:6805/4201493740","state":["exists","up"]},{"osd":1,"uuid":"75cc25e0-d127-4511-8dfa-438ba34f62c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6816","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6817","nonce":1493137965}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6818","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6819","nonce":1493137965}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6822","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6823","nonce":1493137965}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6820","nonce":1493137965},{"type":"v1","addr":"10.20.193.107:6821","nonce":1493137965}]},"public_addr":"10.20.193.107:6817/1493137965","cluster_addr":"10.20.193.107:6819/1493137965","heartbeat_back_addr":"10.20.193.107:6823/1493137965","heartbeat_front_addr":"10.20.193.107:6821/1493137965","state":["exists","up"]},{"osd":2,"uuid":"d25ad5e6-49c5-486e-b22d-e4a8966305d8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6808","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6809","nonce":884104108}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6810","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6811","nonce":884104108}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6814","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6815","nonce":884104108}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.107:6812","nonce":884104108},{"type":"v1","addr":"10.20.193.107:6813","nonce":884104108}]},"public_addr":"10.20.193.107:6809/884104108","cluster_addr":"10.20.193.107:6811/884104108","heartbeat_back_addr":"10.20.193.107:6815/884104108","heartbeat_front_addr":"10.20.193.107:6813/884104108","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:37:35.829971+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:37:35.852049+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:37:35.925364+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.107:0/854378422":"2026-02-20T23:12:58.648046+0000","10.20.193.107:0/3534471284":"2026-02-20T23:11:29.803934+0000","10.20.193.107:0/2678146866":"2026-02-20T23:09:24.846470+0000","10.20.193.107:0/2834058088":"2026-02-20T23:08:04.361034+0000","10.20.193.107:0/1257077562":"2026-02-20T23:07:23.537951+0000","10.20.193.107:0/1547105692":"2026-02-20T23:06:46.152653+0000","10.20.193.107:0/3190832119":"2026-02-20T23:03:21.674902+0000","10.20.193.107:0/246261625":"2026-02-20T22:59:12.287405+0000","10.20.193.107:0/3189971602":"2026-02-20T22:58:48.651886+0000","10.20.193.107:0/205562401":"2026-02-20T22:57:23.367589+0000","10.20.193.107:0/1432314190":"2026-02-20T23:01:45.330345+0000","10.20.193.107:0/2757656806":"2026-02-20T22:58:19.732540+0000","10.20.193.107:0/1543563426":"2026-02-20T22:58:03.459369+0000","10.20.193.107:0/3185978297":"2026-02-20T22:57:31.431758+0000","10.20.193.107:0/2805254962":"2026-02-20T22:58:55.838329+0000","10.20.193.107:0/3744437625":"2026-02-20T22:57:47.428049+0000","10.20.193.107:0/1955764210":"2026-02-20T22:57:15.402529+0000","10.20.193.107:0/610810392":"2026-02-20T22:58:41.633917+0000","10.20.193.107:0/752504233":"2026-02-20T23:02:57.465574+0000","10.20.193.107:0/552737129":"2026-02-20T22:40:17.535143+0000","10.20.193.107:0/2182637050":"2026-02-20T23:12:13.524396+0000","10.20.193.107:0/497553305":"2026-02-20T22:45:24.624745+0000","10.20.193.107:0/3852926790":"2026-02-20T23:00:00.412490+0000","10.20.193.107:0/2272553444":"2026-02-20T22:46:10.350947+0000","10.20.193.107:0/898993945":"2026-02-20T22:56:50.751863+0000","10.20.193.107:0/1313336399":"2026-02-20T22:58:26.600967+0000","10.20.193.107:0/4146850529":"2026-02-20T22:46:55.848573+0000","10.20.193.107:0/2550957406":"2026-02-20T22:41:06.534225+0000","10.20.193.107:0/2390115763":"2026-02-20T22:59:04.305572+0000","10.20.193.107:0/2483163577":"2026-02-20T23:02:09.369122+0000","10.20.193.107:0/2267091544":"2026-02-20T22:57:55.437243+0000","10.20.193.107:0/488876033":"2026-02-20T23:10:48.765255+0000","10.20.193.107:0/3093321588":"2026-02-20T23:08:42.690070+0000","10.20.193.107:0/2862879196":"2026-02-20T22:58:35.256293+0000","10.20.193.107:0/977899992":"2026-02-20T22:41:55.046168+0000","10.20.193.107:0/3233522592":"2026-02-20T23:02:01.580866+0000","10.20.193.107:0/4122333045":"2026-02-20T22:42:47.716855+0000","10.20.193.107:0/879854773":"2026-02-20T23:10:08.706829+0000","10.20.193.107:0/1573977375":"2026-02-20T22:39:30.754966+0000","10.20.193.107:0/1605864109":"2026-02-20T22:56:59.669964+0000","10.20.193.107:0/2365581353":"2026-02-20T22:59:52.407232+0000","10.20.193.107:0/1584298956":"2026-02-20T22:43:37.428378+0000","10.20.193.107:0/1488681155":"2026-02-20T22:59:36.570139+0000","10.20.193.107:0/2770422709":"2026-02-20T23:00:08.609882+0000","10.20.193.107:0/1273542491":"2026-02-20T22:59:44.622992+0000","10.20.193.107:0/769354983":"2026-02-20T22:57:39.424480+0000","10.20.193.107:0/3411501246":"2026-02-20T22:44:28.993471+0000","10.20.193.107:0/3285308326":"2026-02-20T23:03:13.652678+0000","10.20.193.107:0/37138130":"2026-02-20T23:00:24.873032+0000","10.20.193.107:0/1667512935":"2026-02-20T23:00:33.502486+0000","10.20.193.107:0/2801788522":"2026-02-20T22:59:28.352082+0000","10.20.193.107:0/2245716240":"2026-02-20T23:00:41.531437+0000","10.20.193.107:0/4045933686":"2026-02-20T22:59:20.345844+0000","10.20.193.107:0/2846570395":"2026-02-20T23:00:48.635562+0000","10.20.193.107:0/839969197":"2026-02-20T23:00:55.646121+0000","10.20.193.107:0/1072760058":"2026-02-20T23:01:04.304497+0000","10.20.193.107:0/3409712565":"2026-02-20T23:01:12.699636+0000","10.20.193.107:0/3980171512":"2026-02-20T23:01:21.539843+0000","10.20.193.107:0/662155266":"2026-02-20T23:00:16.414276+0000","10.20.193.107:0/2827080139":"2026-02-20T23:01:29.336751+0000","10.20.193.107:0/2675052090":"2026-02-20T23:03:05.460145+0000","10.20.193.107:0/1461265913":"2026-02-20T23:01:37.585830+0000","10.20.193.107:0/332824445":"2026-02-20T22:58:11.458881+0000","10.20.193.107:0/982252671":"2026-02-20T23:01:53.591685+0000","10.20.193.107:0/2613684487":"2026-02-20T23:02:17.392532+0000","10.20.193.107:0/1406670494":"2026-02-20T23:02:41.429660+0000","10.20.193.107:0/4088041825":"2026-02-20T23:02:25.423222+0000","10.20.193.107:0/238843256":"2026-02-20T23:02:33.415969+0000","10.20.193.107:0/4289872299":"2026-02-20T22:57:06.726314+0000","10.20.193.107:0/4150662321":"2026-02-20T23:02:49.643043+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":194,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T22:13:10.518 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-20T22:13:10.518 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-20T22:13:10.604 INFO:teuthology.orchestra.run.trial107.stdout:{ 2026-02-20T22:13:10.604 INFO:teuthology.orchestra.run.trial107.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T22:13:10.604 INFO:teuthology.orchestra.run.trial107.stdout:} 2026-02-20T22:13:10.613 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-20T22:13:10.770 INFO:teuthology.orchestra.run.trial107.stderr:instructed osd(s) 0 to deep-scrub 2026-02-20T22:13:10.783 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-20T22:13:10.783 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-20T22:13:10.859 INFO:teuthology.orchestra.run.trial107.stdout:{ 2026-02-20T22:13:10.859 INFO:teuthology.orchestra.run.trial107.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-20T22:13:10.860 INFO:teuthology.orchestra.run.trial107.stdout:} 2026-02-20T22:13:10.869 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-20T22:13:11.032 INFO:teuthology.orchestra.run.trial107.stderr:instructed osd(s) 1 to deep-scrub 2026-02-20T22:13:11.043 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-20T22:13:11.044 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-20T22:13:11.129 INFO:teuthology.orchestra.run.trial107.stdout:{ 2026-02-20T22:13:11.129 INFO:teuthology.orchestra.run.trial107.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-20T22:13:11.130 INFO:teuthology.orchestra.run.trial107.stdout:} 2026-02-20T22:13:11.141 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-20T22:13:11.292 INFO:teuthology.orchestra.run.trial107.stderr:instructed osd(s) 2 to deep-scrub 2026-02-20T22:13:11.303 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T22:13:11.462 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T22:13:11.463 INFO:teuthology.orchestra.run.trial107.stderr:dumped all 2026-02-20T22:13:11.473 INFO:teuthology.orchestra.run.trial107.stdout:{"pg_ready":true,"pg_map":{"version":1568,"stamp":"2026-02-20T22:13:10.609056+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":227,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4575844,"kb_used_data":17716,"kb_used_omap":1933,"kb_used_meta":4556146,"kb_avail":278539676,"statfs":{"total":289910292480,"available":285224628224,"internally_reserved":0,"allocated":18141184,"data_stored":10164781,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":1979879,"internal_metadata":4665494041},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":19,"apply_latency_ms":19,"commit_latency_ns":19000000,"apply_latency_ns":19000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-536871032,"num_objects":-174,"num_object_clones":0,"num_object_copies":-348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-174,"num_whiteouts":0,"num_read":-136375,"num_read_kb":-39848963,"num_write":-1449189,"num_write_kb":-540169603,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":-1,"num_objects_omap":-18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001247"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.786156+0000","last_change":"2026-02-20T21:37:43.629317+0000","last_active":"2026-02-20T22:13:05.786156+0000","last_peered":"2026-02-20T22:13:05.786156+0000","last_clean":"2026-02-20T22:13:05.786156+0000","last_became_active":"2026-02-20T21:37:41.626252+0000","last_became_peered":"2026-02-20T21:37:41.626252+0000","last_unstale":"2026-02-20T22:13:05.786156+0000","last_undegraded":"2026-02-20T22:13:05.786156+0000","last_fullsized":"2026-02-20T22:13:05.786156+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:57:26.704591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000878194,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.791963+0000","last_change":"2026-02-20T21:37:43.628857+0000","last_active":"2026-02-20T22:13:05.791963+0000","last_peered":"2026-02-20T22:13:05.791963+0000","last_clean":"2026-02-20T22:13:05.791963+0000","last_became_active":"2026-02-20T21:37:41.626370+0000","last_became_peered":"2026-02-20T21:37:41.626370+0000","last_unstale":"2026-02-20T22:13:05.791963+0000","last_undegraded":"2026-02-20T22:13:05.791963+0000","last_fullsized":"2026-02-20T22:13:05.791963+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:39:08.125334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050412799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1014,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.789961+0000","last_change":"2026-02-20T21:37:43.629072+0000","last_active":"2026-02-20T22:13:05.789961+0000","last_peered":"2026-02-20T22:13:05.789961+0000","last_clean":"2026-02-20T22:13:05.789961+0000","last_became_active":"2026-02-20T21:37:41.626503+0000","last_became_peered":"2026-02-20T21:37:41.626503+0000","last_unstale":"2026-02-20T22:13:05.789961+0000","last_undegraded":"2026-02-20T22:13:05.789961+0000","last_fullsized":"2026-02-20T22:13:05.789961+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:55:04.979567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000697728,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1015,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.784329+0000","last_change":"2026-02-20T21:37:43.628823+0000","last_active":"2026-02-20T22:13:05.784329+0000","last_peered":"2026-02-20T22:13:05.784329+0000","last_clean":"2026-02-20T22:13:05.784329+0000","last_became_active":"2026-02-20T21:37:41.626044+0000","last_became_peered":"2026-02-20T21:37:41.626044+0000","last_unstale":"2026-02-20T22:13:05.784329+0000","last_undegraded":"2026-02-20T22:13:05.784329+0000","last_fullsized":"2026-02-20T22:13:05.784329+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:43:21.886202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052464600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":1018,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:08.314216+0000","last_change":"2026-02-20T21:37:43.627134+0000","last_active":"2026-02-20T22:13:08.314216+0000","last_peered":"2026-02-20T22:13:08.314216+0000","last_clean":"2026-02-20T22:13:08.314216+0000","last_became_active":"2026-02-20T21:37:41.626685+0000","last_became_peered":"2026-02-20T21:37:41.626685+0000","last_unstale":"2026-02-20T22:13:08.314216+0000","last_undegraded":"2026-02-20T22:13:08.314216+0000","last_fullsized":"2026-02-20T22:13:08.314216+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:19:24.951160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066038900000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1017,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:07.792337+0000","last_change":"2026-02-20T21:37:43.653844+0000","last_active":"2026-02-20T22:13:07.792337+0000","last_peered":"2026-02-20T22:13:07.792337+0000","last_clean":"2026-02-20T22:13:07.792337+0000","last_became_active":"2026-02-20T21:37:41.626554+0000","last_became_peered":"2026-02-20T21:37:41.626554+0000","last_unstale":"2026-02-20T22:13:07.792337+0000","last_undegraded":"2026-02-20T22:13:07.792337+0000","last_fullsized":"2026-02-20T22:13:07.792337+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:23:54.818688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021567099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1017,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:07.790856+0000","last_change":"2026-02-20T21:37:43.653839+0000","last_active":"2026-02-20T22:13:07.790856+0000","last_peered":"2026-02-20T22:13:07.790856+0000","last_clean":"2026-02-20T22:13:07.790856+0000","last_became_active":"2026-02-20T21:37:41.626692+0000","last_became_peered":"2026-02-20T21:37:41.626692+0000","last_unstale":"2026-02-20T22:13:07.790856+0000","last_undegraded":"2026-02-20T22:13:07.790856+0000","last_fullsized":"2026-02-20T22:13:07.790856+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:10:13.031958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021587200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":1016,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.792266+0000","last_change":"2026-02-20T21:37:43.629180+0000","last_active":"2026-02-20T22:13:05.792266+0000","last_peered":"2026-02-20T22:13:05.792266+0000","last_clean":"2026-02-20T22:13:05.792266+0000","last_became_active":"2026-02-20T21:37:41.626007+0000","last_became_peered":"2026-02-20T21:37:41.626007+0000","last_unstale":"2026-02-20T22:13:05.792266+0000","last_undegraded":"2026-02-20T22:13:05.792266+0000","last_fullsized":"2026-02-20T22:13:05.792266+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:40.616474+0000","last_clean_scrub_stamp":"2026-02-20T21:37:40.616474+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:01:20.786792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080498799999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1104,"reported_epoch":503,"state":"active+clean","last_fresh":"2026-02-20T22:13:05.790346+0000","last_change":"2026-02-20T21:37:39.618505+0000","last_active":"2026-02-20T22:13:05.790346+0000","last_peered":"2026-02-20T22:13:05.790346+0000","last_clean":"2026-02-20T22:13:05.790346+0000","last_became_active":"2026-02-20T21:37:39.618273+0000","last_became_peered":"2026-02-20T21:37:39.618273+0000","last_unstale":"2026-02-20T22:13:05.790346+0000","last_undegraded":"2026-02-20T22:13:05.790346+0000","last_fullsized":"2026-02-20T22:13:05.790346+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:38.611598+0000","last_clean_scrub_stamp":"2026-02-20T21:37:38.611598+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:26:23.564860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738797,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1277368,"kb_used_data":5864,"kb_used_omap":1252,"kb_used_meta":1270235,"kb_avail":93094472,"statfs":{"total":96636764160,"available":95328739328,"internally_reserved":0,"allocated":6004736,"data_stored":2996886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1282996,"internal_metadata":1300720716},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738798,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1841020,"kb_used_data":5932,"kb_used_omap":199,"kb_used_meta":1834872,"kb_avail":92530820,"statfs":{"total":96636764160,"available":94751559680,"internally_reserved":0,"allocated":6074368,"data_stored":3587273,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":203859,"internal_metadata":1878909869},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738796,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1457456,"kb_used_data":5920,"kb_used_omap":481,"kb_used_meta":1451038,"kb_avail":92914384,"statfs":{"total":96636764160,"available":95144329216,"internally_reserved":0,"allocated":6062080,"data_stored":3580622,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":493024,"internal_metadata":1485863456},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:13:11.478 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-20T21:37:40.616474+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=40, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=9, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:11.478 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-20T21:37:40.616474+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=40, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=9, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:11.479 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-20T21:37:40.616474+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=40, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=9, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:11.479 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-20T21:37:40.616474+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=40, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=9, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:11.479 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-20T21:37:40.616474+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=40, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=9, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:11.479 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-20T21:37:40.616474+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=40, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=9, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:11.479 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-20T21:37:40.616474+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=40, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=9, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:11.479 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-20T21:37:40.616474+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=40, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=9, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:11.480 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-20T21:37:38.611598+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=38, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=9, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:11.480 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-20T22:13:31.481 DEBUG:teuthology.orchestra.run.trial107:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T22:13:31.645 INFO:teuthology.orchestra.run.trial107.stdout: 2026-02-20T22:13:31.645 INFO:teuthology.orchestra.run.trial107.stderr:dumped all 2026-02-20T22:13:31.655 INFO:teuthology.orchestra.run.trial107.stdout:{"pg_ready":true,"pg_map":{"version":1578,"stamp":"2026-02-20T22:13:30.610901+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4575856,"kb_used_data":17728,"kb_used_omap":1933,"kb_used_meta":4556146,"kb_avail":278539664,"statfs":{"total":289910292480,"available":285224615936,"internally_reserved":0,"allocated":18153472,"data_stored":10171432,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":1980293,"internal_metadata":4665493627},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001055"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":1024,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:16.623217+0000","last_change":"2026-02-20T22:13:16.623217+0000","last_active":"2026-02-20T22:13:16.623217+0000","last_peered":"2026-02-20T22:13:16.623217+0000","last_clean":"2026-02-20T22:13:16.623217+0000","last_became_active":"2026-02-20T21:37:41.626252+0000","last_became_peered":"2026-02-20T21:37:41.626252+0000","last_unstale":"2026-02-20T22:13:16.623217+0000","last_undegraded":"2026-02-20T22:13:16.623217+0000","last_fullsized":"2026-02-20T22:13:16.623217+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:16.623151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:16.623151+0000","last_clean_scrub_stamp":"2026-02-20T22:13:16.623151+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:09:33.306786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000878194,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1024,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:14.594388+0000","last_change":"2026-02-20T22:13:14.594388+0000","last_active":"2026-02-20T22:13:14.594388+0000","last_peered":"2026-02-20T22:13:14.594388+0000","last_clean":"2026-02-20T22:13:14.594388+0000","last_became_active":"2026-02-20T21:37:41.626370+0000","last_became_peered":"2026-02-20T21:37:41.626370+0000","last_unstale":"2026-02-20T22:13:14.594388+0000","last_undegraded":"2026-02-20T22:13:14.594388+0000","last_fullsized":"2026-02-20T22:13:14.594388+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:14.594303+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:14.594303+0000","last_clean_scrub_stamp":"2026-02-20T22:13:14.594303+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:38:05.522752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050412799999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1025,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:15.597351+0000","last_change":"2026-02-20T22:13:15.597287+0000","last_active":"2026-02-20T22:13:15.597351+0000","last_peered":"2026-02-20T22:13:15.597351+0000","last_clean":"2026-02-20T22:13:15.597351+0000","last_became_active":"2026-02-20T21:37:41.626503+0000","last_became_peered":"2026-02-20T21:37:41.626503+0000","last_unstale":"2026-02-20T22:13:15.597351+0000","last_undegraded":"2026-02-20T22:13:15.597351+0000","last_fullsized":"2026-02-20T22:13:15.597351+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:15.597228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:15.597228+0000","last_clean_scrub_stamp":"2026-02-20T22:13:15.597228+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:50:56.048954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000697728,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1025,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:13.614355+0000","last_change":"2026-02-20T22:13:13.614355+0000","last_active":"2026-02-20T22:13:13.614355+0000","last_peered":"2026-02-20T22:13:13.614355+0000","last_clean":"2026-02-20T22:13:13.614355+0000","last_became_active":"2026-02-20T21:37:41.626044+0000","last_became_peered":"2026-02-20T21:37:41.626044+0000","last_unstale":"2026-02-20T22:13:13.614355+0000","last_undegraded":"2026-02-20T22:13:13.614355+0000","last_fullsized":"2026-02-20T22:13:13.614355+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:13.614277+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:13.614277+0000","last_clean_scrub_stamp":"2026-02-20T22:13:13.614277+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:35:35.866389+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0.00052464600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"504'3","reported_seq":1027,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:10.887164+0000","last_change":"2026-02-20T22:13:10.887164+0000","last_active":"2026-02-20T22:13:10.887164+0000","last_peered":"2026-02-20T22:13:10.887164+0000","last_clean":"2026-02-20T22:13:10.887164+0000","last_became_active":"2026-02-20T21:37:41.626685+0000","last_became_peered":"2026-02-20T21:37:41.626685+0000","last_unstale":"2026-02-20T22:13:10.887164+0000","last_undegraded":"2026-02-20T22:13:10.887164+0000","last_fullsized":"2026-02-20T22:13:10.887164+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'3","last_scrub_stamp":"2026-02-20T22:13:10.887134+0000","last_deep_scrub":"504'3","last_deep_scrub_stamp":"2026-02-20T22:13:10.887134+0000","last_clean_scrub_stamp":"2026-02-20T22:13:10.887134+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:02:19.378885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066038900000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1025,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:12.651917+0000","last_change":"2026-02-20T22:13:12.651917+0000","last_active":"2026-02-20T22:13:12.651917+0000","last_peered":"2026-02-20T22:13:12.651917+0000","last_clean":"2026-02-20T22:13:12.651917+0000","last_became_active":"2026-02-20T21:37:41.626554+0000","last_became_peered":"2026-02-20T21:37:41.626554+0000","last_unstale":"2026-02-20T22:13:12.651917+0000","last_undegraded":"2026-02-20T22:13:12.651917+0000","last_fullsized":"2026-02-20T22:13:12.651917+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:12.651845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:12.651845+0000","last_clean_scrub_stamp":"2026-02-20T22:13:12.651845+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:58:42.807582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021567099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1025,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:11.607268+0000","last_change":"2026-02-20T22:13:11.607268+0000","last_active":"2026-02-20T22:13:11.607268+0000","last_peered":"2026-02-20T22:13:11.607268+0000","last_clean":"2026-02-20T22:13:11.607268+0000","last_became_active":"2026-02-20T21:37:41.626692+0000","last_became_peered":"2026-02-20T21:37:41.626692+0000","last_unstale":"2026-02-20T22:13:11.607268+0000","last_undegraded":"2026-02-20T22:13:11.607268+0000","last_fullsized":"2026-02-20T22:13:11.607268+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:11.607208+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:11.607208+0000","last_clean_scrub_stamp":"2026-02-20T22:13:11.607208+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:14:07.663734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021587200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"504'2","reported_seq":1027,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:12.600044+0000","last_change":"2026-02-20T22:13:12.600044+0000","last_active":"2026-02-20T22:13:12.600044+0000","last_peered":"2026-02-20T22:13:12.600044+0000","last_clean":"2026-02-20T22:13:12.600044+0000","last_became_active":"2026-02-20T21:37:41.626007+0000","last_became_peered":"2026-02-20T21:37:41.626007+0000","last_unstale":"2026-02-20T22:13:12.600044+0000","last_undegraded":"2026-02-20T22:13:12.600044+0000","last_fullsized":"2026-02-20T22:13:12.600044+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'2","last_scrub_stamp":"2026-02-20T22:13:12.599988+0000","last_deep_scrub":"504'2","last_deep_scrub_stamp":"2026-02-20T22:13:12.599988+0000","last_clean_scrub_stamp":"2026-02-20T22:13:12.599988+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:48:01.020683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080498799999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"504'74","reported_seq":1116,"reported_epoch":504,"state":"active+clean","last_fresh":"2026-02-20T22:13:11.571013+0000","last_change":"2026-02-20T22:13:11.571013+0000","last_active":"2026-02-20T22:13:11.571013+0000","last_peered":"2026-02-20T22:13:11.571013+0000","last_clean":"2026-02-20T22:13:11.571013+0000","last_became_active":"2026-02-20T21:37:39.618273+0000","last_became_peered":"2026-02-20T21:37:39.618273+0000","last_unstale":"2026-02-20T22:13:11.571013+0000","last_undegraded":"2026-02-20T22:13:11.571013+0000","last_fullsized":"2026-02-20T22:13:11.571013+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"504'74","last_scrub_stamp":"2026-02-20T22:13:11.570956+0000","last_deep_scrub":"504'74","last_deep_scrub_stamp":"2026-02-20T22:13:11.570956+0000","last_clean_scrub_stamp":"2026-02-20T22:13:11.570956+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:44:31.909717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67202,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738801,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1277368,"kb_used_data":5864,"kb_used_omap":1252,"kb_used_meta":1270235,"kb_avail":93094472,"statfs":{"total":96636764160,"available":95328739328,"internally_reserved":0,"allocated":6004736,"data_stored":2996886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1282996,"internal_metadata":1300720716},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738802,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1841020,"kb_used_data":5932,"kb_used_omap":199,"kb_used_meta":1834872,"kb_avail":92530820,"statfs":{"total":96636764160,"available":94751559680,"internally_reserved":0,"allocated":6074368,"data_stored":3587273,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":203859,"internal_metadata":1878909869},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738800,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1457468,"kb_used_data":5932,"kb_used_omap":481,"kb_used_meta":1451038,"kb_avail":92914372,"statfs":{"total":96636764160,"available":95144316928,"internally_reserved":0,"allocated":6074368,"data_stored":3587273,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":493438,"internal_metadata":1485863042},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33601,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:13:31.658 DEBUG:teuthology.orchestra.run.trial107:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-20T22:13:31.822 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-20T22:13:31.822 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-20T22:13:31.822 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-20T22:13:31.822 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:35.984 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 is failed for ~0s 2026-02-20T22:13:37.924 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-20T22:13:37.924 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-20T22:13:37.925 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:41.288 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.1 is failed for ~0s 2026-02-20T22:13:41.288 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.0 has been restored 2026-02-20T22:13:44.026 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-20T22:13:44.027 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-20T22:13:44.027 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:46.490 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 is failed for ~0s 2026-02-20T22:13:46.490 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.1 has been restored 2026-02-20T22:13:50.129 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-20T22:13:50.129 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-20T22:13:50.130 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-20T22:13:50.130 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:50.176 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-20T22:13:50.177 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-20T22:13:50.177 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-20T22:13:50.177 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:13:50.195 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-20T22:13:50.195 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-20T22:13:50.196 DEBUG:teuthology.orchestra.run.trial107:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-20T22:13:50.248 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial107.front.sepia.ceph.com 2026-02-20T22:13:50.248 DEBUG:teuthology.orchestra.run.trial107:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-20T22:13:50.490 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial107.front.sepia.ceph.com 2026-02-20T22:13:50.490 DEBUG:teuthology.orchestra.run.trial107:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-20T22:13:50.651 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial107.front.sepia.ceph.com 2026-02-20T22:13:50.652 DEBUG:teuthology.orchestra.run.trial107:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-20T22:13:50.815 INFO:tasks.ceph:Archiving mon data... 2026-02-20T22:13:50.816 DEBUG:teuthology.misc:Transferring archived files from trial107:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486/data/cluster2.mon.a.tgz 2026-02-20T22:13:50.816 DEBUG:teuthology.orchestra.run.trial107:> mktemp 2026-02-20T22:13:50.821 INFO:teuthology.orchestra.run.trial107.stdout:/tmp/tmp.LVjpbbpSE3 2026-02-20T22:13:50.822 DEBUG:teuthology.orchestra.run.trial107:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.LVjpbbpSE3 2026-02-20T22:13:50.935 DEBUG:teuthology.orchestra.run.trial107:> sudo chmod 0666 /tmp/tmp.LVjpbbpSE3 2026-02-20T22:13:50.995 DEBUG:teuthology.orchestra.remote:trial107:/tmp/tmp.LVjpbbpSE3 is 473KB 2026-02-20T22:13:51.055 DEBUG:teuthology.orchestra.run.trial107:> rm -fr /tmp/tmp.LVjpbbpSE3 2026-02-20T22:13:51.060 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-20T22:13:51.060 DEBUG:teuthology.orchestra.run.trial032:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T22:13:51.062 DEBUG:teuthology.orchestra.run.trial107:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T22:13:51.109 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-20T22:13:51.131 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-20T22:13:51.131 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T22:13:51.290 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T22:13:51.291 INFO:teuthology.orchestra.run.trial032.stderr:dumped all 2026-02-20T22:13:51.301 INFO:teuthology.orchestra.run.trial032.stdout:{"pg_ready":true,"pg_map":{"version":1561,"stamp":"2026-02-20T22:13:49.940312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":398936,"kb_used_data":11368,"kb_used_omap":8138,"kb_used_meta":379381,"kb_avail":282716584,"statfs":{"total":289910292480,"available":289501782016,"internally_reserved":0,"allocated":11640832,"data_stored":7346635,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":8333930,"internal_metadata":388486550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000873"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":939,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.740034+0000","last_change":"2026-02-20T21:37:20.673446+0000","last_active":"2026-02-20T22:13:06.740034+0000","last_peered":"2026-02-20T22:13:06.740034+0000","last_clean":"2026-02-20T22:13:06.740034+0000","last_became_active":"2026-02-20T21:37:18.668090+0000","last_became_peered":"2026-02-20T21:37:18.668090+0000","last_unstale":"2026-02-20T22:13:06.740034+0000","last_undegraded":"2026-02-20T22:13:06.740034+0000","last_fullsized":"2026-02-20T22:13:06.740034+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:30:26.985444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038342900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":939,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.742116+0000","last_change":"2026-02-20T21:37:20.673605+0000","last_active":"2026-02-20T22:13:06.742116+0000","last_peered":"2026-02-20T22:13:06.742116+0000","last_clean":"2026-02-20T22:13:06.742116+0000","last_became_active":"2026-02-20T21:37:18.668149+0000","last_became_peered":"2026-02-20T21:37:18.668149+0000","last_unstale":"2026-02-20T22:13:06.742116+0000","last_undegraded":"2026-02-20T22:13:06.742116+0000","last_fullsized":"2026-02-20T22:13:06.742116+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:13:26.573716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055130999999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":940,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741500+0000","last_change":"2026-02-20T21:37:20.673738+0000","last_active":"2026-02-20T22:13:06.741500+0000","last_peered":"2026-02-20T22:13:06.741500+0000","last_clean":"2026-02-20T22:13:06.741500+0000","last_became_active":"2026-02-20T21:37:18.667922+0000","last_became_peered":"2026-02-20T21:37:18.667922+0000","last_unstale":"2026-02-20T22:13:06.741500+0000","last_undegraded":"2026-02-20T22:13:06.741500+0000","last_fullsized":"2026-02-20T22:13:06.741500+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:19:51.832309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065968000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":940,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741380+0000","last_change":"2026-02-20T21:37:20.673640+0000","last_active":"2026-02-20T22:13:06.741380+0000","last_peered":"2026-02-20T22:13:06.741380+0000","last_clean":"2026-02-20T22:13:06.741380+0000","last_became_active":"2026-02-20T21:37:18.668675+0000","last_became_peered":"2026-02-20T21:37:18.668675+0000","last_unstale":"2026-02-20T22:13:06.741380+0000","last_undegraded":"2026-02-20T22:13:06.741380+0000","last_fullsized":"2026-02-20T22:13:06.741380+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:14:58.246769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058640500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":943,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.738468+0000","last_change":"2026-02-20T21:37:20.671740+0000","last_active":"2026-02-20T22:13:06.738468+0000","last_peered":"2026-02-20T22:13:06.738468+0000","last_clean":"2026-02-20T22:13:06.738468+0000","last_became_active":"2026-02-20T21:37:18.668118+0000","last_became_peered":"2026-02-20T21:37:18.668118+0000","last_unstale":"2026-02-20T22:13:06.738468+0000","last_undegraded":"2026-02-20T22:13:06.738468+0000","last_fullsized":"2026-02-20T22:13:06.738468+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:38:22.882998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068900500000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":941,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.738851+0000","last_change":"2026-02-20T21:37:20.668922+0000","last_active":"2026-02-20T22:13:06.738851+0000","last_peered":"2026-02-20T22:13:06.738851+0000","last_clean":"2026-02-20T22:13:06.738851+0000","last_became_active":"2026-02-20T21:37:18.668095+0000","last_became_peered":"2026-02-20T21:37:18.668095+0000","last_unstale":"2026-02-20T22:13:06.738851+0000","last_undegraded":"2026-02-20T22:13:06.738851+0000","last_fullsized":"2026-02-20T22:13:06.738851+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:26:41.579853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045394000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":942,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.739918+0000","last_change":"2026-02-20T21:37:20.668873+0000","last_active":"2026-02-20T22:13:06.739918+0000","last_peered":"2026-02-20T22:13:06.739918+0000","last_clean":"2026-02-20T22:13:06.739918+0000","last_became_active":"2026-02-20T21:37:18.668650+0000","last_became_peered":"2026-02-20T21:37:18.668650+0000","last_unstale":"2026-02-20T22:13:06.739918+0000","last_undegraded":"2026-02-20T22:13:06.739918+0000","last_fullsized":"2026-02-20T22:13:06.739918+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:09:53.022016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040528899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":941,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741028+0000","last_change":"2026-02-20T21:37:20.673439+0000","last_active":"2026-02-20T22:13:06.741028+0000","last_peered":"2026-02-20T22:13:06.741028+0000","last_clean":"2026-02-20T22:13:06.741028+0000","last_became_active":"2026-02-20T21:37:18.667168+0000","last_became_peered":"2026-02-20T21:37:18.667168+0000","last_unstale":"2026-02-20T22:13:06.741028+0000","last_undegraded":"2026-02-20T22:13:06.741028+0000","last_fullsized":"2026-02-20T22:13:06.741028+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:25:34.870707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037955200000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1030,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.742335+0000","last_change":"2026-02-20T21:37:16.895581+0000","last_active":"2026-02-20T22:13:06.742335+0000","last_peered":"2026-02-20T22:13:06.742335+0000","last_clean":"2026-02-20T22:13:06.742335+0000","last_became_active":"2026-02-20T21:37:16.894669+0000","last_became_peered":"2026-02-20T21:37:16.894669+0000","last_unstale":"2026-02-20T22:13:06.742335+0000","last_undegraded":"2026-02-20T22:13:06.742335+0000","last_fullsized":"2026-02-20T22:13:06.742335+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_clean_scrub_stamp":"2026-02-20T21:37:15.654907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:21:36.071811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738809,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120880,"kb_used_data":3744,"kb_used_omap":3044,"kb_used_meta":114075,"kb_avail":94250960,"statfs":{"total":96636764160,"available":96512983040,"internally_reserved":0,"allocated":3833856,"data_stored":2055287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3117096,"internal_metadata":116813784},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120948,"kb_used_data":3812,"kb_used_omap":2087,"kb_used_meta":115032,"kb_avail":94250892,"statfs":{"total":96636764160,"available":96512913408,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2138017,"internal_metadata":117792863},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738810,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":157108,"kb_used_data":3812,"kb_used_omap":3006,"kb_used_meta":150273,"kb_avail":94214732,"statfs":{"total":96636764160,"available":96475885568,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":3078817,"internal_metadata":153879903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:13:51.302 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T22:13:51.461 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T22:13:51.462 INFO:teuthology.orchestra.run.trial032.stderr:dumped all 2026-02-20T22:13:51.473 INFO:teuthology.orchestra.run.trial032.stdout:{"pg_ready":true,"pg_map":{"version":1561,"stamp":"2026-02-20T22:13:49.940312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":398936,"kb_used_data":11368,"kb_used_omap":8138,"kb_used_meta":379381,"kb_avail":282716584,"statfs":{"total":289910292480,"available":289501782016,"internally_reserved":0,"allocated":11640832,"data_stored":7346635,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":8333930,"internal_metadata":388486550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000873"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":939,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.740034+0000","last_change":"2026-02-20T21:37:20.673446+0000","last_active":"2026-02-20T22:13:06.740034+0000","last_peered":"2026-02-20T22:13:06.740034+0000","last_clean":"2026-02-20T22:13:06.740034+0000","last_became_active":"2026-02-20T21:37:18.668090+0000","last_became_peered":"2026-02-20T21:37:18.668090+0000","last_unstale":"2026-02-20T22:13:06.740034+0000","last_undegraded":"2026-02-20T22:13:06.740034+0000","last_fullsized":"2026-02-20T22:13:06.740034+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:30:26.985444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038342900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":939,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.742116+0000","last_change":"2026-02-20T21:37:20.673605+0000","last_active":"2026-02-20T22:13:06.742116+0000","last_peered":"2026-02-20T22:13:06.742116+0000","last_clean":"2026-02-20T22:13:06.742116+0000","last_became_active":"2026-02-20T21:37:18.668149+0000","last_became_peered":"2026-02-20T21:37:18.668149+0000","last_unstale":"2026-02-20T22:13:06.742116+0000","last_undegraded":"2026-02-20T22:13:06.742116+0000","last_fullsized":"2026-02-20T22:13:06.742116+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:13:26.573716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055130999999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":940,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741500+0000","last_change":"2026-02-20T21:37:20.673738+0000","last_active":"2026-02-20T22:13:06.741500+0000","last_peered":"2026-02-20T22:13:06.741500+0000","last_clean":"2026-02-20T22:13:06.741500+0000","last_became_active":"2026-02-20T21:37:18.667922+0000","last_became_peered":"2026-02-20T21:37:18.667922+0000","last_unstale":"2026-02-20T22:13:06.741500+0000","last_undegraded":"2026-02-20T22:13:06.741500+0000","last_fullsized":"2026-02-20T22:13:06.741500+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:19:51.832309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065968000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":940,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741380+0000","last_change":"2026-02-20T21:37:20.673640+0000","last_active":"2026-02-20T22:13:06.741380+0000","last_peered":"2026-02-20T22:13:06.741380+0000","last_clean":"2026-02-20T22:13:06.741380+0000","last_became_active":"2026-02-20T21:37:18.668675+0000","last_became_peered":"2026-02-20T21:37:18.668675+0000","last_unstale":"2026-02-20T22:13:06.741380+0000","last_undegraded":"2026-02-20T22:13:06.741380+0000","last_fullsized":"2026-02-20T22:13:06.741380+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:14:58.246769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058640500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":943,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.738468+0000","last_change":"2026-02-20T21:37:20.671740+0000","last_active":"2026-02-20T22:13:06.738468+0000","last_peered":"2026-02-20T22:13:06.738468+0000","last_clean":"2026-02-20T22:13:06.738468+0000","last_became_active":"2026-02-20T21:37:18.668118+0000","last_became_peered":"2026-02-20T21:37:18.668118+0000","last_unstale":"2026-02-20T22:13:06.738468+0000","last_undegraded":"2026-02-20T22:13:06.738468+0000","last_fullsized":"2026-02-20T22:13:06.738468+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:38:22.882998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068900500000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":941,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.738851+0000","last_change":"2026-02-20T21:37:20.668922+0000","last_active":"2026-02-20T22:13:06.738851+0000","last_peered":"2026-02-20T22:13:06.738851+0000","last_clean":"2026-02-20T22:13:06.738851+0000","last_became_active":"2026-02-20T21:37:18.668095+0000","last_became_peered":"2026-02-20T21:37:18.668095+0000","last_unstale":"2026-02-20T22:13:06.738851+0000","last_undegraded":"2026-02-20T22:13:06.738851+0000","last_fullsized":"2026-02-20T22:13:06.738851+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:26:41.579853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045394000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":942,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.739918+0000","last_change":"2026-02-20T21:37:20.668873+0000","last_active":"2026-02-20T22:13:06.739918+0000","last_peered":"2026-02-20T22:13:06.739918+0000","last_clean":"2026-02-20T22:13:06.739918+0000","last_became_active":"2026-02-20T21:37:18.668650+0000","last_became_peered":"2026-02-20T21:37:18.668650+0000","last_unstale":"2026-02-20T22:13:06.739918+0000","last_undegraded":"2026-02-20T22:13:06.739918+0000","last_fullsized":"2026-02-20T22:13:06.739918+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:09:53.022016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040528899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":941,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741028+0000","last_change":"2026-02-20T21:37:20.673439+0000","last_active":"2026-02-20T22:13:06.741028+0000","last_peered":"2026-02-20T22:13:06.741028+0000","last_clean":"2026-02-20T22:13:06.741028+0000","last_became_active":"2026-02-20T21:37:18.667168+0000","last_became_peered":"2026-02-20T21:37:18.667168+0000","last_unstale":"2026-02-20T22:13:06.741028+0000","last_undegraded":"2026-02-20T22:13:06.741028+0000","last_fullsized":"2026-02-20T22:13:06.741028+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:25:34.870707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037955200000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1030,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.742335+0000","last_change":"2026-02-20T21:37:16.895581+0000","last_active":"2026-02-20T22:13:06.742335+0000","last_peered":"2026-02-20T22:13:06.742335+0000","last_clean":"2026-02-20T22:13:06.742335+0000","last_became_active":"2026-02-20T21:37:16.894669+0000","last_became_peered":"2026-02-20T21:37:16.894669+0000","last_unstale":"2026-02-20T22:13:06.742335+0000","last_undegraded":"2026-02-20T22:13:06.742335+0000","last_fullsized":"2026-02-20T22:13:06.742335+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_clean_scrub_stamp":"2026-02-20T21:37:15.654907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:21:36.071811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738809,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120880,"kb_used_data":3744,"kb_used_omap":3044,"kb_used_meta":114075,"kb_avail":94250960,"statfs":{"total":96636764160,"available":96512983040,"internally_reserved":0,"allocated":3833856,"data_stored":2055287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3117096,"internal_metadata":116813784},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120948,"kb_used_data":3812,"kb_used_omap":2087,"kb_used_meta":115032,"kb_avail":94250892,"statfs":{"total":96636764160,"available":96512913408,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2138017,"internal_metadata":117792863},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738810,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":157108,"kb_used_data":3812,"kb_used_omap":3006,"kb_used_meta":150273,"kb_avail":94214732,"statfs":{"total":96636764160,"available":96475885568,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":3078817,"internal_metadata":153879903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:13:51.475 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-20T22:13:51.475 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T22:13:51.491 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster2.osd.2 has been restored 2026-02-20T22:13:51.635 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T22:13:51.635 INFO:teuthology.orchestra.run.trial032.stderr:dumped all 2026-02-20T22:13:51.645 INFO:teuthology.orchestra.run.trial032.stdout:{"pg_ready":true,"pg_map":{"version":1561,"stamp":"2026-02-20T22:13:49.940312+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":398936,"kb_used_data":11368,"kb_used_omap":8138,"kb_used_meta":379381,"kb_avail":282716584,"statfs":{"total":289910292480,"available":289501782016,"internally_reserved":0,"allocated":11640832,"data_stored":7346635,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":8333930,"internal_metadata":388486550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000873"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":939,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.740034+0000","last_change":"2026-02-20T21:37:20.673446+0000","last_active":"2026-02-20T22:13:06.740034+0000","last_peered":"2026-02-20T22:13:06.740034+0000","last_clean":"2026-02-20T22:13:06.740034+0000","last_became_active":"2026-02-20T21:37:18.668090+0000","last_became_peered":"2026-02-20T21:37:18.668090+0000","last_unstale":"2026-02-20T22:13:06.740034+0000","last_undegraded":"2026-02-20T22:13:06.740034+0000","last_fullsized":"2026-02-20T22:13:06.740034+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:30:26.985444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038342900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":939,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.742116+0000","last_change":"2026-02-20T21:37:20.673605+0000","last_active":"2026-02-20T22:13:06.742116+0000","last_peered":"2026-02-20T22:13:06.742116+0000","last_clean":"2026-02-20T22:13:06.742116+0000","last_became_active":"2026-02-20T21:37:18.668149+0000","last_became_peered":"2026-02-20T21:37:18.668149+0000","last_unstale":"2026-02-20T22:13:06.742116+0000","last_undegraded":"2026-02-20T22:13:06.742116+0000","last_fullsized":"2026-02-20T22:13:06.742116+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:13:26.573716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055130999999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":940,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741500+0000","last_change":"2026-02-20T21:37:20.673738+0000","last_active":"2026-02-20T22:13:06.741500+0000","last_peered":"2026-02-20T22:13:06.741500+0000","last_clean":"2026-02-20T22:13:06.741500+0000","last_became_active":"2026-02-20T21:37:18.667922+0000","last_became_peered":"2026-02-20T21:37:18.667922+0000","last_unstale":"2026-02-20T22:13:06.741500+0000","last_undegraded":"2026-02-20T22:13:06.741500+0000","last_fullsized":"2026-02-20T22:13:06.741500+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:19:51.832309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065968000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":940,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741380+0000","last_change":"2026-02-20T21:37:20.673640+0000","last_active":"2026-02-20T22:13:06.741380+0000","last_peered":"2026-02-20T22:13:06.741380+0000","last_clean":"2026-02-20T22:13:06.741380+0000","last_became_active":"2026-02-20T21:37:18.668675+0000","last_became_peered":"2026-02-20T21:37:18.668675+0000","last_unstale":"2026-02-20T22:13:06.741380+0000","last_undegraded":"2026-02-20T22:13:06.741380+0000","last_fullsized":"2026-02-20T22:13:06.741380+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:14:58.246769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058640500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":943,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.738468+0000","last_change":"2026-02-20T21:37:20.671740+0000","last_active":"2026-02-20T22:13:06.738468+0000","last_peered":"2026-02-20T22:13:06.738468+0000","last_clean":"2026-02-20T22:13:06.738468+0000","last_became_active":"2026-02-20T21:37:18.668118+0000","last_became_peered":"2026-02-20T21:37:18.668118+0000","last_unstale":"2026-02-20T22:13:06.738468+0000","last_undegraded":"2026-02-20T22:13:06.738468+0000","last_fullsized":"2026-02-20T22:13:06.738468+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:38:22.882998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068900500000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":941,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.738851+0000","last_change":"2026-02-20T21:37:20.668922+0000","last_active":"2026-02-20T22:13:06.738851+0000","last_peered":"2026-02-20T22:13:06.738851+0000","last_clean":"2026-02-20T22:13:06.738851+0000","last_became_active":"2026-02-20T21:37:18.668095+0000","last_became_peered":"2026-02-20T21:37:18.668095+0000","last_unstale":"2026-02-20T22:13:06.738851+0000","last_undegraded":"2026-02-20T22:13:06.738851+0000","last_fullsized":"2026-02-20T22:13:06.738851+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:26:41.579853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045394000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":942,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.739918+0000","last_change":"2026-02-20T21:37:20.668873+0000","last_active":"2026-02-20T22:13:06.739918+0000","last_peered":"2026-02-20T22:13:06.739918+0000","last_clean":"2026-02-20T22:13:06.739918+0000","last_became_active":"2026-02-20T21:37:18.668650+0000","last_became_peered":"2026-02-20T21:37:18.668650+0000","last_unstale":"2026-02-20T22:13:06.739918+0000","last_undegraded":"2026-02-20T22:13:06.739918+0000","last_fullsized":"2026-02-20T22:13:06.739918+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:09:53.022016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040528899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":941,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741028+0000","last_change":"2026-02-20T21:37:20.673439+0000","last_active":"2026-02-20T22:13:06.741028+0000","last_peered":"2026-02-20T22:13:06.741028+0000","last_clean":"2026-02-20T22:13:06.741028+0000","last_became_active":"2026-02-20T21:37:18.667168+0000","last_became_peered":"2026-02-20T21:37:18.667168+0000","last_unstale":"2026-02-20T22:13:06.741028+0000","last_undegraded":"2026-02-20T22:13:06.741028+0000","last_fullsized":"2026-02-20T22:13:06.741028+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:25:34.870707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037955200000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1030,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.742335+0000","last_change":"2026-02-20T21:37:16.895581+0000","last_active":"2026-02-20T22:13:06.742335+0000","last_peered":"2026-02-20T22:13:06.742335+0000","last_clean":"2026-02-20T22:13:06.742335+0000","last_became_active":"2026-02-20T21:37:16.894669+0000","last_became_peered":"2026-02-20T21:37:16.894669+0000","last_unstale":"2026-02-20T22:13:06.742335+0000","last_undegraded":"2026-02-20T22:13:06.742335+0000","last_fullsized":"2026-02-20T22:13:06.742335+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_clean_scrub_stamp":"2026-02-20T21:37:15.654907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:21:36.071811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738809,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120880,"kb_used_data":3744,"kb_used_omap":3044,"kb_used_meta":114075,"kb_avail":94250960,"statfs":{"total":96636764160,"available":96512983040,"internally_reserved":0,"allocated":3833856,"data_stored":2055287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3117096,"internal_metadata":116813784},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738809,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120948,"kb_used_data":3812,"kb_used_omap":2087,"kb_used_meta":115032,"kb_avail":94250892,"statfs":{"total":96636764160,"available":96512913408,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2138017,"internal_metadata":117792863},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738810,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":157108,"kb_used_data":3812,"kb_used_omap":3006,"kb_used_meta":150273,"kb_avail":94214732,"statfs":{"total":96636764160,"available":96475885568,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":3078817,"internal_metadata":153879903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:13:51.647 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T22:13:51.794 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T22:13:51.795 INFO:teuthology.orchestra.run.trial032.stdout:{"epoch":466,"fsid":"637d54fe-a5fc-4c51-9045-58e4626774a2","created":"2026-02-20T21:37:09.505649+0000","modified":"2026-02-20T22:13:06.731772+0000","last_up_change":"2026-02-20T21:37:14.652696+0000","last_in_change":"2026-02-20T21:37:10.129240+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:37:15.649121+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:37:17.481675+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"14","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":14,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"d4d665db-e2ee-45c1-a7d0-483f617a59af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6800","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6801","nonce":2386430400}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6802","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6803","nonce":2386430400}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6806","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6807","nonce":2386430400}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6804","nonce":2386430400},{"type":"v1","addr":"10.20.193.32:6805","nonce":2386430400}]},"public_addr":"10.20.193.32:6801/2386430400","cluster_addr":"10.20.193.32:6803/2386430400","heartbeat_back_addr":"10.20.193.32:6807/2386430400","heartbeat_front_addr":"10.20.193.32:6805/2386430400","state":["exists","up"]},{"osd":1,"uuid":"f94266a3-dac8-4ba7-b0be-47a485fc51d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6808","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6809","nonce":128603091}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6810","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6811","nonce":128603091}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6814","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6815","nonce":128603091}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6812","nonce":128603091},{"type":"v1","addr":"10.20.193.32:6813","nonce":128603091}]},"public_addr":"10.20.193.32:6809/128603091","cluster_addr":"10.20.193.32:6811/128603091","heartbeat_back_addr":"10.20.193.32:6815/128603091","heartbeat_front_addr":"10.20.193.32:6813/128603091","state":["exists","up"]},{"osd":2,"uuid":"15a644f0-20e2-46b2-9a46-894f8dcf405c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":28,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6816","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6817","nonce":2752957932}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6818","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6819","nonce":2752957932}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6822","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6823","nonce":2752957932}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.32:6820","nonce":2752957932},{"type":"v1","addr":"10.20.193.32:6821","nonce":2752957932}]},"public_addr":"10.20.193.32:6817/2752957932","cluster_addr":"10.20.193.32:6819/2752957932","heartbeat_back_addr":"10.20.193.32:6823/2752957932","heartbeat_front_addr":"10.20.193.32:6821/2752957932","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:37:12.947825+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:37:13.099940+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:37:13.139627+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":197,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T22:13:52.808 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-20T22:13:52.808 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-20T22:13:52.884 INFO:teuthology.orchestra.run.trial032.stdout:{ 2026-02-20T22:13:52.885 INFO:teuthology.orchestra.run.trial032.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T22:13:52.885 INFO:teuthology.orchestra.run.trial032.stdout:} 2026-02-20T22:13:52.894 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-20T22:13:53.047 INFO:teuthology.orchestra.run.trial032.stderr:instructed osd(s) 0 to deep-scrub 2026-02-20T22:13:53.060 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-20T22:13:53.060 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-20T22:13:53.147 INFO:teuthology.orchestra.run.trial032.stdout:{ 2026-02-20T22:13:53.147 INFO:teuthology.orchestra.run.trial032.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T22:13:53.148 INFO:teuthology.orchestra.run.trial032.stdout:} 2026-02-20T22:13:53.160 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-20T22:13:53.318 INFO:teuthology.orchestra.run.trial032.stderr:instructed osd(s) 1 to deep-scrub 2026-02-20T22:13:53.329 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-20T22:13:53.329 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-20T22:13:53.410 INFO:teuthology.orchestra.run.trial032.stdout:{ 2026-02-20T22:13:53.410 INFO:teuthology.orchestra.run.trial032.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T22:13:53.410 INFO:teuthology.orchestra.run.trial032.stdout:} 2026-02-20T22:13:53.420 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-20T22:13:53.571 INFO:teuthology.orchestra.run.trial032.stderr:instructed osd(s) 2 to deep-scrub 2026-02-20T22:13:53.584 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T22:13:53.745 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T22:13:53.746 INFO:teuthology.orchestra.run.trial032.stderr:dumped all 2026-02-20T22:13:53.756 INFO:teuthology.orchestra.run.trial032.stdout:{"pg_ready":true,"pg_map":{"version":1562,"stamp":"2026-02-20T22:13:51.940467+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":398936,"kb_used_data":11368,"kb_used_omap":8138,"kb_used_meta":379381,"kb_avail":282716584,"statfs":{"total":289910292480,"available":289501782016,"internally_reserved":0,"allocated":11640832,"data_stored":7346635,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":8333930,"internal_metadata":388486550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000906"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":939,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.740034+0000","last_change":"2026-02-20T21:37:20.673446+0000","last_active":"2026-02-20T22:13:06.740034+0000","last_peered":"2026-02-20T22:13:06.740034+0000","last_clean":"2026-02-20T22:13:06.740034+0000","last_became_active":"2026-02-20T21:37:18.668090+0000","last_became_peered":"2026-02-20T21:37:18.668090+0000","last_unstale":"2026-02-20T22:13:06.740034+0000","last_undegraded":"2026-02-20T22:13:06.740034+0000","last_fullsized":"2026-02-20T22:13:06.740034+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:30:26.985444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038342900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":939,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.742116+0000","last_change":"2026-02-20T21:37:20.673605+0000","last_active":"2026-02-20T22:13:06.742116+0000","last_peered":"2026-02-20T22:13:06.742116+0000","last_clean":"2026-02-20T22:13:06.742116+0000","last_became_active":"2026-02-20T21:37:18.668149+0000","last_became_peered":"2026-02-20T21:37:18.668149+0000","last_unstale":"2026-02-20T22:13:06.742116+0000","last_undegraded":"2026-02-20T22:13:06.742116+0000","last_fullsized":"2026-02-20T22:13:06.742116+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:13:26.573716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055130999999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":940,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741500+0000","last_change":"2026-02-20T21:37:20.673738+0000","last_active":"2026-02-20T22:13:06.741500+0000","last_peered":"2026-02-20T22:13:06.741500+0000","last_clean":"2026-02-20T22:13:06.741500+0000","last_became_active":"2026-02-20T21:37:18.667922+0000","last_became_peered":"2026-02-20T21:37:18.667922+0000","last_unstale":"2026-02-20T22:13:06.741500+0000","last_undegraded":"2026-02-20T22:13:06.741500+0000","last_fullsized":"2026-02-20T22:13:06.741500+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:19:51.832309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065968000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":940,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741380+0000","last_change":"2026-02-20T21:37:20.673640+0000","last_active":"2026-02-20T22:13:06.741380+0000","last_peered":"2026-02-20T22:13:06.741380+0000","last_clean":"2026-02-20T22:13:06.741380+0000","last_became_active":"2026-02-20T21:37:18.668675+0000","last_became_peered":"2026-02-20T21:37:18.668675+0000","last_unstale":"2026-02-20T22:13:06.741380+0000","last_undegraded":"2026-02-20T22:13:06.741380+0000","last_fullsized":"2026-02-20T22:13:06.741380+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:14:58.246769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058640500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"14'2","reported_seq":943,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.738468+0000","last_change":"2026-02-20T21:37:20.671740+0000","last_active":"2026-02-20T22:13:06.738468+0000","last_peered":"2026-02-20T22:13:06.738468+0000","last_clean":"2026-02-20T22:13:06.738468+0000","last_became_active":"2026-02-20T21:37:18.668118+0000","last_became_peered":"2026-02-20T21:37:18.668118+0000","last_unstale":"2026-02-20T22:13:06.738468+0000","last_undegraded":"2026-02-20T22:13:06.738468+0000","last_fullsized":"2026-02-20T22:13:06.738468+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:38:22.882998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068900500000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":941,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.738851+0000","last_change":"2026-02-20T21:37:20.668922+0000","last_active":"2026-02-20T22:13:06.738851+0000","last_peered":"2026-02-20T22:13:06.738851+0000","last_clean":"2026-02-20T22:13:06.738851+0000","last_became_active":"2026-02-20T21:37:18.668095+0000","last_became_peered":"2026-02-20T21:37:18.668095+0000","last_unstale":"2026-02-20T22:13:06.738851+0000","last_undegraded":"2026-02-20T22:13:06.738851+0000","last_fullsized":"2026-02-20T22:13:06.738851+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:26:41.579853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045394000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":942,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.739918+0000","last_change":"2026-02-20T21:37:20.668873+0000","last_active":"2026-02-20T22:13:06.739918+0000","last_peered":"2026-02-20T22:13:06.739918+0000","last_clean":"2026-02-20T22:13:06.739918+0000","last_became_active":"2026-02-20T21:37:18.668650+0000","last_became_peered":"2026-02-20T21:37:18.668650+0000","last_unstale":"2026-02-20T22:13:06.739918+0000","last_undegraded":"2026-02-20T22:13:06.739918+0000","last_fullsized":"2026-02-20T22:13:06.739918+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:09:53.022016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040528899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"12'1","reported_seq":941,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.741028+0000","last_change":"2026-02-20T21:37:20.673439+0000","last_active":"2026-02-20T22:13:06.741028+0000","last_peered":"2026-02-20T22:13:06.741028+0000","last_clean":"2026-02-20T22:13:06.741028+0000","last_became_active":"2026-02-20T21:37:18.667168+0000","last_became_peered":"2026-02-20T21:37:18.667168+0000","last_unstale":"2026-02-20T22:13:06.741028+0000","last_undegraded":"2026-02-20T22:13:06.741028+0000","last_fullsized":"2026-02-20T22:13:06.741028+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:17.659703+0000","last_clean_scrub_stamp":"2026-02-20T21:37:17.659703+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:25:34.870707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037955200000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":1030,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:06.742335+0000","last_change":"2026-02-20T21:37:16.895581+0000","last_active":"2026-02-20T22:13:06.742335+0000","last_peered":"2026-02-20T22:13:06.742335+0000","last_clean":"2026-02-20T22:13:06.742335+0000","last_became_active":"2026-02-20T21:37:16.894669+0000","last_became_peered":"2026-02-20T21:37:16.894669+0000","last_unstale":"2026-02-20T22:13:06.742335+0000","last_undegraded":"2026-02-20T22:13:06.742335+0000","last_fullsized":"2026-02-20T22:13:06.742335+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:37:15.654907+0000","last_clean_scrub_stamp":"2026-02-20T21:37:15.654907+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:21:36.071811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738810,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120880,"kb_used_data":3744,"kb_used_omap":3044,"kb_used_meta":114075,"kb_avail":94250960,"statfs":{"total":96636764160,"available":96512983040,"internally_reserved":0,"allocated":3833856,"data_stored":2055287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3117096,"internal_metadata":116813784},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738810,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120948,"kb_used_data":3812,"kb_used_omap":2087,"kb_used_meta":115032,"kb_avail":94250892,"statfs":{"total":96636764160,"available":96512913408,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2138017,"internal_metadata":117792863},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738810,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":157108,"kb_used_data":3812,"kb_used_omap":3006,"kb_used_meta":150273,"kb_avail":94214732,"statfs":{"total":96636764160,"available":96475885568,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":3078817,"internal_metadata":153879903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:13:53.757 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-20T21:37:17.659703+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=17, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:53.758 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-20T21:37:17.659703+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=17, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:53.758 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-20T21:37:17.659703+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=17, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:53.758 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-20T21:37:17.659703+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=17, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:53.758 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-20T21:37:17.659703+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=17, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:53.758 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-20T21:37:17.659703+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=17, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:53.758 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-20T21:37:17.659703+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=17, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:53.759 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-20T21:37:17.659703+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=17, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:53.759 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-20T21:37:15.654907+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=37, tm_sec=15, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=22, tm_min=13, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T22:13:53.759 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-20T22:14:13.760 DEBUG:teuthology.orchestra.run.trial032:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T22:14:13.917 INFO:teuthology.orchestra.run.trial032.stdout: 2026-02-20T22:14:13.917 INFO:teuthology.orchestra.run.trial032.stderr:dumped all 2026-02-20T22:14:13.927 INFO:teuthology.orchestra.run.trial032.stdout:{"pg_ready":true,"pg_map":{"version":1572,"stamp":"2026-02-20T22:14:11.942325+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":398936,"kb_used_data":11368,"kb_used_omap":8138,"kb_used_meta":379381,"kb_avail":282716584,"statfs":{"total":289910292480,"available":289501782016,"internally_reserved":0,"allocated":11640832,"data_stored":7346635,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":8333930,"internal_metadata":388486550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001034"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":948,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:58.384655+0000","last_change":"2026-02-20T22:13:58.384594+0000","last_active":"2026-02-20T22:13:58.384655+0000","last_peered":"2026-02-20T22:13:58.384655+0000","last_clean":"2026-02-20T22:13:58.384655+0000","last_became_active":"2026-02-20T21:37:18.668090+0000","last_became_peered":"2026-02-20T21:37:18.668090+0000","last_unstale":"2026-02-20T22:13:58.384655+0000","last_undegraded":"2026-02-20T22:13:58.384655+0000","last_fullsized":"2026-02-20T22:13:58.384655+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:58.384540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:58.384540+0000","last_clean_scrub_stamp":"2026-02-20T22:13:58.384540+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:49:37.719796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038342900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":948,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:56.428718+0000","last_change":"2026-02-20T22:13:56.428676+0000","last_active":"2026-02-20T22:13:56.428718+0000","last_peered":"2026-02-20T22:13:56.428718+0000","last_clean":"2026-02-20T22:13:56.428718+0000","last_became_active":"2026-02-20T21:37:18.668149+0000","last_became_peered":"2026-02-20T21:37:18.668149+0000","last_unstale":"2026-02-20T22:13:56.428718+0000","last_undegraded":"2026-02-20T22:13:56.428718+0000","last_fullsized":"2026-02-20T22:13:56.428718+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:56.428621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:56.428621+0000","last_clean_scrub_stamp":"2026-02-20T22:13:56.428621+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:49:04.349533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055130999999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":948,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:57.384838+0000","last_change":"2026-02-20T22:13:57.384838+0000","last_active":"2026-02-20T22:13:57.384838+0000","last_peered":"2026-02-20T22:13:57.384838+0000","last_clean":"2026-02-20T22:13:57.384838+0000","last_became_active":"2026-02-20T21:37:18.667922+0000","last_became_peered":"2026-02-20T21:37:18.667922+0000","last_unstale":"2026-02-20T22:13:57.384838+0000","last_undegraded":"2026-02-20T22:13:57.384838+0000","last_fullsized":"2026-02-20T22:13:57.384838+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:57.384774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:57.384774+0000","last_clean_scrub_stamp":"2026-02-20T22:13:57.384774+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:21:50.328308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065968000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":948,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:55.436769+0000","last_change":"2026-02-20T22:13:55.436769+0000","last_active":"2026-02-20T22:13:55.436769+0000","last_peered":"2026-02-20T22:13:55.436769+0000","last_clean":"2026-02-20T22:13:55.436769+0000","last_became_active":"2026-02-20T21:37:18.668675+0000","last_became_peered":"2026-02-20T21:37:18.668675+0000","last_unstale":"2026-02-20T22:13:55.436769+0000","last_undegraded":"2026-02-20T22:13:55.436769+0000","last_fullsized":"2026-02-20T22:13:55.436769+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:55.436723+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:55.436723+0000","last_clean_scrub_stamp":"2026-02-20T22:13:55.436723+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:34:51.028907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058640500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"466'3","reported_seq":952,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:53.454890+0000","last_change":"2026-02-20T22:13:53.454890+0000","last_active":"2026-02-20T22:13:53.454890+0000","last_peered":"2026-02-20T22:13:53.454890+0000","last_clean":"2026-02-20T22:13:53.454890+0000","last_became_active":"2026-02-20T21:37:18.668118+0000","last_became_peered":"2026-02-20T21:37:18.668118+0000","last_unstale":"2026-02-20T22:13:53.454890+0000","last_undegraded":"2026-02-20T22:13:53.454890+0000","last_fullsized":"2026-02-20T22:13:53.454890+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"466'3","last_scrub_stamp":"2026-02-20T22:13:53.454847+0000","last_deep_scrub":"466'3","last_deep_scrub_stamp":"2026-02-20T22:13:53.454847+0000","last_clean_scrub_stamp":"2026-02-20T22:13:53.454847+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:31:14.172370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068900500000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":949,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:54.801073+0000","last_change":"2026-02-20T22:13:54.801073+0000","last_active":"2026-02-20T22:13:54.801073+0000","last_peered":"2026-02-20T22:13:54.801073+0000","last_clean":"2026-02-20T22:13:54.801073+0000","last_became_active":"2026-02-20T21:37:18.668095+0000","last_became_peered":"2026-02-20T21:37:18.668095+0000","last_unstale":"2026-02-20T22:13:54.801073+0000","last_undegraded":"2026-02-20T22:13:54.801073+0000","last_fullsized":"2026-02-20T22:13:54.801073+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:54.800975+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:54.800975+0000","last_clean_scrub_stamp":"2026-02-20T22:13:54.800975+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:10:38.016319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045394000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":950,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:53.756604+0000","last_change":"2026-02-20T22:13:53.756604+0000","last_active":"2026-02-20T22:13:53.756604+0000","last_peered":"2026-02-20T22:13:53.756604+0000","last_clean":"2026-02-20T22:13:53.756604+0000","last_became_active":"2026-02-20T21:37:18.668650+0000","last_became_peered":"2026-02-20T21:37:18.668650+0000","last_unstale":"2026-02-20T22:13:53.756604+0000","last_undegraded":"2026-02-20T22:13:53.756604+0000","last_fullsized":"2026-02-20T22:13:53.756604+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T22:13:53.756538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T22:13:53.756538+0000","last_clean_scrub_stamp":"2026-02-20T22:13:53.756538+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:15:18.587815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040528899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"466'2","reported_seq":950,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:54.464223+0000","last_change":"2026-02-20T22:13:54.464223+0000","last_active":"2026-02-20T22:13:54.464223+0000","last_peered":"2026-02-20T22:13:54.464223+0000","last_clean":"2026-02-20T22:13:54.464223+0000","last_became_active":"2026-02-20T21:37:18.667168+0000","last_became_peered":"2026-02-20T21:37:18.667168+0000","last_unstale":"2026-02-20T22:13:54.464223+0000","last_undegraded":"2026-02-20T22:13:54.464223+0000","last_fullsized":"2026-02-20T22:13:54.464223+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"466'2","last_scrub_stamp":"2026-02-20T22:13:54.464138+0000","last_deep_scrub":"466'2","last_deep_scrub_stamp":"2026-02-20T22:13:54.464138+0000","last_clean_scrub_stamp":"2026-02-20T22:13:54.464138+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:36:04.738712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037955200000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"466'74","reported_seq":1040,"reported_epoch":466,"state":"active+clean","last_fresh":"2026-02-20T22:13:53.426235+0000","last_change":"2026-02-20T22:13:53.426235+0000","last_active":"2026-02-20T22:13:53.426235+0000","last_peered":"2026-02-20T22:13:53.426235+0000","last_clean":"2026-02-20T22:13:53.426235+0000","last_became_active":"2026-02-20T21:37:16.894669+0000","last_became_peered":"2026-02-20T21:37:16.894669+0000","last_unstale":"2026-02-20T22:13:53.426235+0000","last_undegraded":"2026-02-20T22:13:53.426235+0000","last_fullsized":"2026-02-20T22:13:53.426235+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"466'74","last_scrub_stamp":"2026-02-20T22:13:53.426148+0000","last_deep_scrub":"466'74","last_deep_scrub_stamp":"2026-02-20T22:13:53.426148+0000","last_clean_scrub_stamp":"2026-02-20T22:13:53.426148+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:22:02.531103+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":1180736,"data_compressed":67218,"data_compressed_allocated":90112,"data_compressed_original":1146880,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738814,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120880,"kb_used_data":3744,"kb_used_omap":3044,"kb_used_meta":114075,"kb_avail":94250960,"statfs":{"total":96636764160,"available":96512983040,"internally_reserved":0,"allocated":3833856,"data_stored":2055287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3117096,"internal_metadata":116813784},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738814,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":120948,"kb_used_data":3812,"kb_used_omap":2087,"kb_used_meta":115032,"kb_avail":94250892,"statfs":{"total":96636764160,"available":96512913408,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":2138017,"internal_metadata":117792863},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738814,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":157108,"kb_used_data":3812,"kb_used_omap":3006,"kb_used_meta":150273,"kb_avail":94214732,"statfs":{"total":96636764160,"available":96475885568,"internally_reserved":0,"allocated":3903488,"data_stored":2645674,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":3078817,"internal_metadata":153879903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":590368,"data_compressed":33609,"data_compressed_allocated":45056,"data_compressed_original":573440,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T22:14:13.929 DEBUG:teuthology.orchestra.run.trial032:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-20T22:14:14.094 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-20T22:14:14.094 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-20T22:14:14.094 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-20T22:14:14.094 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:14:19.052 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 is failed for ~0s 2026-02-20T22:14:20.196 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-20T22:14:20.197 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-20T22:14:20.197 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:14:20.296 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-20T22:14:20.296 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-20T22:14:20.296 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:14:20.391 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-20T22:14:20.391 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-20T22:14:20.392 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-20T22:14:20.392 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:14:20.419 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-20T22:14:20.419 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-20T22:14:20.419 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-20T22:14:20.419 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T22:14:20.477 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-20T22:14:20.477 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-20T22:14:20.478 DEBUG:teuthology.orchestra.run.trial032:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-20T22:14:20.528 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial032.front.sepia.ceph.com 2026-02-20T22:14:20.528 DEBUG:teuthology.orchestra.run.trial032:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-20T22:14:20.589 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial032.front.sepia.ceph.com 2026-02-20T22:14:20.589 DEBUG:teuthology.orchestra.run.trial032:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-20T22:14:20.654 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial032.front.sepia.ceph.com 2026-02-20T22:14:20.655 DEBUG:teuthology.orchestra.run.trial032:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-20T22:14:20.719 INFO:tasks.ceph:Archiving mon data... 2026-02-20T22:14:20.720 DEBUG:teuthology.misc:Transferring archived files from trial032:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486/data/cluster1.mon.a.tgz 2026-02-20T22:14:20.720 DEBUG:teuthology.orchestra.run.trial032:> mktemp 2026-02-20T22:14:20.765 INFO:teuthology.orchestra.run.trial032.stdout:/tmp/tmp.xEv61wEZj7 2026-02-20T22:14:20.766 DEBUG:teuthology.orchestra.run.trial032:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.xEv61wEZj7 2026-02-20T22:14:20.864 DEBUG:teuthology.orchestra.run.trial032:> sudo chmod 0666 /tmp/tmp.xEv61wEZj7 2026-02-20T22:14:20.923 DEBUG:teuthology.orchestra.remote:trial032:/tmp/tmp.xEv61wEZj7 is 399KB 2026-02-20T22:14:20.979 DEBUG:teuthology.orchestra.run.trial032:> rm -fr /tmp/tmp.xEv61wEZj7 2026-02-20T22:14:20.982 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-20T22:14:20.982 DEBUG:teuthology.orchestra.run.trial032:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T22:14:21.026 DEBUG:teuthology.orchestra.run.trial107:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T22:14:21.078 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-20T22:14:21.082 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-20T22:14:21.082 INFO:tasks.ceph:Archiving crash dumps... 2026-02-20T22:14:21.086 DEBUG:teuthology.misc:Transferring archived files from trial032:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486/remote/trial032/crash 2026-02-20T22:14:21.087 DEBUG:teuthology.orchestra.run.trial032:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-20T22:14:21.126 DEBUG:teuthology.misc:Transferring archived files from trial107:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486/remote/trial107/crash 2026-02-20T22:14:21.127 DEBUG:teuthology.orchestra.run.trial107:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-20T22:14:21.136 INFO:tasks.ceph:Compressing logs... 2026-02-20T22:14:21.136 DEBUG:teuthology.orchestra.run.trial032:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T22:14:21.171 DEBUG:teuthology.orchestra.run.trial107:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T22:14:21.178 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19048.log 2026-02-20T22:14:21.178 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-20T22:14:21.179 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.19048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18821.log 2026-02-20T22:14:21.179 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19048.log.gz 2026-02-20T22:14:21.179 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-20T22:14:21.179 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-mon.a.log: /var/log/ceph/cluster1-client.admin.18821.log: gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-20T22:14:21.179 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18821.log.gz 2026-02-20T22:14:21.180 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20935.log 2026-02-20T22:14:21.180 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18640.log 2026-02-20T22:14:21.180 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.20935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20883.log 2026-02-20T22:14:21.180 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20935.log.gz 2026-02-20T22:14:21.181 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18640.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18974.log 2026-02-20T22:14:21.181 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18640.log.gz 2026-02-20T22:14:21.181 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log/var/log/ceph/cluster1-client.admin.20883.log: 2026-02-20T22:14:21.181 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20883.log.gz 2026-02-20T22:14:21.181 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18974.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18974.log.gz 2026-02-20T22:14:21.181 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-20T22:14:21.181 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18822.log 2026-02-20T22:14:21.182 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-mgr.x.log: /var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18898.log 2026-02-20T22:14:21.182 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18822.log: 93.0% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-20T22:14:21.182 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18981.log 2026-02-20T22:14:21.182 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18822.log.gz 2026-02-20T22:14:21.183 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18690.log 2026-02-20T22:14:21.183 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18898.log: /var/log/ceph/cluster1-client.admin.18981.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18898.log.gz 2026-02-20T22:14:21.183 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18981.log.gz 2026-02-20T22:14:21.183 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-20T22:14:21.183 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18690.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21091.log 2026-02-20T22:14:21.183 INFO:teuthology.orchestra.run.trial032.stderr: 80.6% -- replaced with /var/log/ceph/cluster1-client.admin.18690.log.gz 2026-02-20T22:14:21.183 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18736.log 2026-02-20T22:14:21.184 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.21091.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16050.log 2026-02-20T22:14:21.184 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21091.log.gz 2026-02-20T22:14:21.184 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21169.log 2026-02-20T22:14:21.184 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18736.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18736.log.gz 2026-02-20T22:14:21.184 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20961.log 2026-02-20T22:14:21.184 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.16050.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16050.log.gz 2026-02-20T22:14:21.185 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.21169.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21169.log.gz 2026-02-20T22:14:21.185 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20909.log 2026-02-20T22:14:21.185 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.20961.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18788.log 2026-02-20T22:14:21.185 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20961.log.gz 2026-02-20T22:14:21.185 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.20909.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18665.log 2026-02-20T22:14:21.185 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20909.log.gz 2026-02-20T22:14:21.186 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15779.log 2026-02-20T22:14:21.186 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18788.log.gz 2026-02-20T22:14:21.186 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25718.log 2026-02-20T22:14:21.186 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18665.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15976.log 2026-02-20T22:14:21.186 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18665.log.gz 2026-02-20T22:14:21.186 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33448.log 2026-02-20T22:14:21.187 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.15779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18900.log 2026-02-20T22:14:21.187 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15779.log.gz 2026-02-20T22:14:21.187 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53861.log 2026-02-20T22:14:21.187 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.15976.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15941.log 2026-02-20T22:14:21.187 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15976.log.gz 2026-02-20T22:14:21.187 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18900.log: 87.8% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-20T22:14:21.187 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18900.log.gz 2026-02-20T22:14:21.187 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21065.log 2026-02-20T22:14:21.188 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19074.log 2026-02-20T22:14:21.188 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.15941.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15941.log.gz 2026-02-20T22:14:21.188 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.21065.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21065.log.gz 2026-02-20T22:14:21.188 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19100.log 2026-02-20T22:14:21.188 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28490.log 2026-02-20T22:14:21.188 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37633.log 2026-02-20T22:14:21.188 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53861.log.gz 2026-02-20T22:14:21.188 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28490.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36669.log 2026-02-20T22:14:21.188 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.33448.log.gz 2026-02-20T22:14:21.189 INFO:teuthology.orchestra.run.trial107.stderr: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28490.log.gz 2026-02-20T22:14:21.189 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27179.log 2026-02-20T22:14:21.189 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36669.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55006.log 2026-02-20T22:14:21.189 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37633.log.gz 2026-02-20T22:14:21.189 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36669.log.gz 2026-02-20T22:14:21.189 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.27179.log: /var/log/ceph/cluster2-client.mirror.32658.log 2026-02-20T22:14:21.189 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55006.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55006.log.gz 2026-02-20T22:14:21.189 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46967.log 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.19074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19074.log.gz 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18820.log 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.19100.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19100.log.gz 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15917.log 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18820.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21143.log 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18820.log.gz 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21117.log 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.15917.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15917.log.gz 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.21143.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21143.log.gz 2026-02-20T22:14:21.190 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21039.log 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18899.log 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.21117.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21117.log.gz 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.21039.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18762.log 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21039.log.gz 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18899.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18899.log.gz 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21193.log 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18762.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18762.log.gz 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21013.log 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.21193.log: /var/log/ceph/cluster1-osd.2.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21193.log.gz 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15893.log 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32658.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23946.log 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46967.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30429.log 2026-02-20T22:14:21.192 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.25718.log.gz 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50761.log 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46967.log.gz 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23946.log: /var/log/ceph/cluster2-client.mirror.30429.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32085.log 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster2-client.mirror.23946.log.gz 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30429.log.gz 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41848.log 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50761.log.gz 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32085.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53053.log 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr: 86.1% -- replaced with /var/log/ceph/cluster2-client.mirror.32085.log.gz 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41134.log 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41848.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27179.log.gz 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53053.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26701.log 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr: 83.4% -- replaced with /var/log/ceph/cluster1-client.mirror.53053.log.gz 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41134.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.41848.log.gz 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26015.log 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26701.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28865.log 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51865.log 2026-02-20T22:14:21.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26015.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41134.log.gz 2026-02-20T22:14:21.194 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27351.log 2026-02-20T22:14:21.194 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.26015.log.gz 2026-02-20T22:14:21.194 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37216.log 2026-02-20T22:14:21.194 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51865.log: /var/log/ceph/cluster2-client.mirror.27351.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51865.log.gz 2026-02-20T22:14:21.194 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39413.log 2026-02-20T22:14:21.194 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25388.log/var/log/ceph/cluster1-client.mirror.37216.log: 2026-02-20T22:14:21.194 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.27351.log.gz 2026-02-20T22:14:21.194 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.39413.log: /var/log/ceph/cluster2-client.mirror.29309.log 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.21013.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18973.log 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.21013.log.gz 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.15893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20987.log 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15893.log.gz 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18973.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.12160.log 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18973.log.gz 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18710.log 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.20987.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20987.log.gz 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/ceph.tmp-client.admin.12160.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.12160.log.gz 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.15851.log 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.18710.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18710.log.gz 2026-02-20T22:14:21.195 INFO:teuthology.orchestra.run.trial032.stderr:/var/log/ceph/cluster1-client.admin.15851.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.15851.log.gz 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.39413.log.gz 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.25388.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34713.log 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26701.log.gz 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21424.log 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29309.log: /var/log/ceph/cluster2-client.mirror.34713.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19385.log 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.25388.log.gz 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21424.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.37216.log.gz 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.34713.log.gz 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37819.log 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.21424.log.gz 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29309.log.gz 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.19385.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28349.log 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr: 85.0% -- replaced with /var/log/ceph/cluster2-client.mirror.19385.log.gz 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42742.log 2026-02-20T22:14:21.196 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37819.log: /var/log/ceph/cluster2-client.mirror.28349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48218.log 2026-02-20T22:14:21.197 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42742.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37819.log.gz 2026-02-20T22:14:21.197 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42694.log 2026-02-20T22:14:21.197 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.42742.log.gz 2026-02-20T22:14:21.197 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28349.log.gz 2026-02-20T22:14:21.197 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48218.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20815.log 2026-02-20T22:14:21.197 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26615.log 2026-02-20T22:14:21.197 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42694.log.gz 2026-02-20T22:14:21.197 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.20815.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22584.log 2026-02-20T22:14:21.197 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41041.log 2026-02-20T22:14:21.198 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26615.log: /var/log/ceph/cluster2-client.mirror.22584.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43019.log 2026-02-20T22:14:21.198 INFO:teuthology.orchestra.run.trial107.stderr:gzip/var/log/ceph/cluster1-client.mirror.41041.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28369.log 2026-02-20T22:14:21.198 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48218.log.gz 2026-02-20T22:14:21.198 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.22584.log.gz 2026-02-20T22:14:21.198 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43019.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41041.log.gz 2026-02-20T22:14:21.198 INFO:teuthology.orchestra.run.trial107.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43019.log.gz 2026-02-20T22:14:21.198 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26850.log 2026-02-20T22:14:21.199 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25941.log 2026-02-20T22:14:21.199 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.28369.log.gz 2026-02-20T22:14:21.199 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26850.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47823.log 2026-02-20T22:14:21.199 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25941.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39896.log 2026-02-20T22:14:21.200 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38487.log 2026-02-20T22:14:21.200 INFO:teuthology.orchestra.run.trial107.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.25941.log.gz 2026-02-20T22:14:21.200 INFO:teuthology.orchestra.run.trial107.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47823.log.gz 2026-02-20T22:14:21.200 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24062.log 2026-02-20T22:14:21.200 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39896.log: /var/log/ceph/cluster1-client.mirror.38487.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.23946.log 2026-02-20T22:14:21.200 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39896.log.gz 2026-02-20T22:14:21.201 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.24062.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24756.log 2026-02-20T22:14:21.201 INFO:teuthology.orchestra.run.trial107.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.24062.log.gz 2026-02-20T22:14:21.201 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.38487.log.gz 2026-02-20T22:14:21.201 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.0.23946.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22454.log 2026-02-20T22:14:21.201 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24756.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26615.log.gz 2026-02-20T22:14:21.201 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29124.log 2026-02-20T22:14:21.201 INFO:teuthology.orchestra.run.trial107.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.24756.log.gz 2026-02-20T22:14:21.202 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22454.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48786.log 2026-02-20T22:14:21.202 INFO:teuthology.orchestra.run.trial107.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.22454.log.gz 2026-02-20T22:14:21.202 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.29848.log 2026-02-20T22:14:21.202 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29124.log: /var/log/ceph/cluster2-client.mirror.48786.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29124.log.gz 2026-02-20T22:14:21.203 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34024.log 2026-02-20T22:14:21.203 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48786.log.gz 2026-02-20T22:14:21.203 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26850.log.gz 2026-02-20T22:14:21.203 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.0.29848.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24610.log 2026-02-20T22:14:21.203 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34024.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32943.log 2026-02-20T22:14:21.203 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.34024.log.gz 2026-02-20T22:14:21.203 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36693.log 2026-02-20T22:14:21.204 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24610.log: /var/log/ceph/cluster1-client.mirror.32943.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.32943.log.gz 2026-02-20T22:14:21.204 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27982.log 2026-02-20T22:14:21.204 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50806.log/var/log/ceph/cluster1-client.mirror.36693.log: 2026-02-20T22:14:21.204 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27982.log: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.23946.log.gz 2026-02-20T22:14:21.204 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36693.log.gz 2026-02-20T22:14:21.204 INFO:teuthology.orchestra.run.trial107.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24610.log.gz 2026-02-20T22:14:21.204 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28411.log 2026-02-20T22:14:21.205 INFO:teuthology.orchestra.run.trial107.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.29848.log.gz 2026-02-20T22:14:21.205 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50806.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43088.log 2026-02-20T22:14:21.205 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50806.log.gz 2026-02-20T22:14:21.205 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19464.log/var/log/ceph/cluster2-client.mirror.28411.log: 2026-02-20T22:14:21.205 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28411.log.gz 2026-02-20T22:14:21.205 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43088.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43826.log 2026-02-20T22:14:21.206 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31530.log 2026-02-20T22:14:21.206 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19464.log: /var/log/ceph/cluster1-client.mirror.43826.log: 83.9% -- replaced with /var/log/ceph/cluster1-client.mirror.19464.log.gz 2026-02-20T22:14:21.206 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54075.log 2026-02-20T22:14:21.206 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43826.log.gz 2026-02-20T22:14:21.206 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31530.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27902.log 2026-02-20T22:14:21.207 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54075.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43088.log.gz 2026-02-20T22:14:21.207 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54552.log 2026-02-20T22:14:21.207 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.31530.log.gz 2026-02-20T22:14:21.207 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27902.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27843.log 2026-02-20T22:14:21.208 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54552.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53782.log 2026-02-20T22:14:21.208 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27843.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54552.log.gz 2026-02-20T22:14:21.208 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43636.log 2026-02-20T22:14:21.208 INFO:teuthology.orchestra.run.trial107.stderr: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27843.log.gz 2026-02-20T22:14:21.208 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53782.log: 91.4%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.54075.log.gz 2026-02-20T22:14:21.209 INFO:teuthology.orchestra.run.trial107.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46194.log 2026-02-20T22:14:21.209 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43636.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53782.log.gz 2026-02-20T22:14:21.209 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28162.log 2026-02-20T22:14:21.209 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46194.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21919.log 2026-02-20T22:14:21.210 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46194.log.gz 89.9% 2026-02-20T22:14:21.210 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.43636.log.gz 2026-02-20T22:14:21.210 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28162.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48600.log 2026-02-20T22:14:21.210 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28162.log.gz 2026-02-20T22:14:21.210 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40607.log 2026-02-20T22:14:21.210 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.21919.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35891.log 2026-02-20T22:14:21.210 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48600.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.21919.log.gz 2026-02-20T22:14:21.211 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40607.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40607.log.gz 2026-02-20T22:14:21.211 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25761.log 2026-02-20T22:14:21.211 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48600.log.gz 2026-02-20T22:14:21.211 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54980.log 2026-02-20T22:14:21.211 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35891.log: /var/log/ceph/cluster1-client.mirror.25761.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51757.log 2026-02-20T22:14:21.211 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35891.log.gz 2026-02-20T22:14:21.211 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.54980.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27551.log 2026-02-20T22:14:21.211 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.54980.log.gz 2026-02-20T22:14:21.212 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51757.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22519.log 2026-02-20T22:14:21.212 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27551.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51757.log.gz 2026-02-20T22:14:21.212 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24382.log 2026-02-20T22:14:21.212 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27551.log.gz 2026-02-20T22:14:21.213 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35313.log 2026-02-20T22:14:21.213 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24382.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41872.log 2026-02-20T22:14:21.213 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52824.log 2026-02-20T22:14:21.213 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35313.log: /var/log/ceph/cluster1-client.mirror.41872.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.25761.log.gz 2026-02-20T22:14:21.214 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41872.log.gz 2026-02-20T22:14:21.214 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38446.log 2026-02-20T22:14:21.214 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35313.log.gz 2026-02-20T22:14:21.214 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36277.log 2026-02-20T22:14:21.214 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52824.log: /var/log/ceph/cluster1-client.mirror.38446.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52824.log.gz 2026-02-20T22:14:21.214 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34691.log 2026-02-20T22:14:21.215 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20039.log 2026-02-20T22:14:21.215 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.38446.log.gz 2026-02-20T22:14:21.215 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34691.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.36277.log.gz 2026-02-20T22:14:21.215 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54621.log 2026-02-20T22:14:21.215 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.20039.log: 65.1% -- replaced with /var/log/ceph/cluster2-client.mirror.20039.log.gz 2026-02-20T22:14:21.215 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28527.log 2026-02-20T22:14:21.215 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.34691.log.gz 2026-02-20T22:14:21.216 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.22519.log.gz 2026-02-20T22:14:21.216 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54621.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29351.log 2026-02-20T22:14:21.216 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.54621.log.gz 2026-02-20T22:14:21.216 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28527.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45402.log 2026-02-20T22:14:21.216 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.28527.log.gz 2026-02-20T22:14:21.216 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19512.log 2026-02-20T22:14:21.217 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45402.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33313.log 2026-02-20T22:14:21.217 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50311.log 2026-02-20T22:14:21.217 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19512.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45402.log.gz 2026-02-20T22:14:21.217 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33313.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.19512.log.gz 2026-02-20T22:14:21.217 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29371.log 2026-02-20T22:14:21.217 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33313.log.gz 2026-02-20T22:14:21.218 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50311.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31044.log 2026-02-20T22:14:21.218 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50311.log.gz 2026-02-20T22:14:21.218 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29371.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29388.log 2026-02-20T22:14:21.218 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29371.log.gz 2026-02-20T22:14:21.218 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45709.log 2026-02-20T22:14:21.219 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31044.log: /var/log/ceph/cluster2-client.mirror.29388.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46512.log 2026-02-20T22:14:21.219 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45709.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33518.log 2026-02-20T22:14:21.219 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29388.log.gz 2026-02-20T22:14:21.219 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31044.log.gz 2026-02-20T22:14:21.219 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.46512.log: --verbose -- /var/log/ceph/cluster1-client.mirror.33132.log 2026-02-20T22:14:21.220 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33518.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27222.log 2026-02-20T22:14:21.220 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.33518.log.gz 2026-02-20T22:14:21.220 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39160.log 2026-02-20T22:14:21.220 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33132.log: /var/log/ceph/cluster2-client.mirror.27222.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45709.log.gz 2026-02-20T22:14:21.220 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45592.log 2026-02-20T22:14:21.220 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33132.log.gz 2026-02-20T22:14:21.221 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39160.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25869.log 2026-02-20T22:14:21.221 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45592.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39160.log.gz 2026-02-20T22:14:21.221 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52689.log 2026-02-20T22:14:21.221 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46512.log.gz 2026-02-20T22:14:21.221 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45592.log.gz 2026-02-20T22:14:21.221 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.25869.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19087.log 2026-02-20T22:14:21.222 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52689.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.25869.log.gz 2026-02-20T22:14:21.222 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33912.log 2026-02-20T22:14:21.222 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19087.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19087.log.gz 2026-02-20T22:14:21.222 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27222.log.gz 2026-02-20T22:14:21.222 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23472.log 2026-02-20T22:14:21.223 INFO:teuthology.orchestra.run.trial107.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52689.log.gz 2026-02-20T22:14:21.223 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33912.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29667.log 2026-02-20T22:14:21.223 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.23472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40694.log 2026-02-20T22:14:21.223 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29667.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.23472.log.gz 2026-02-20T22:14:21.223 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49793.log 2026-02-20T22:14:21.224 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.47299.log 2026-02-20T22:14:21.224 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31152.log 2026-02-20T22:14:21.224 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49793.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33912.log.gz 2026-02-20T22:14:21.224 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.3.47299.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49793.log.gz 2026-02-20T22:14:21.224 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40694.log.gz 2026-02-20T22:14:21.224 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46988.log 2026-02-20T22:14:21.225 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31152.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41800.log 2026-02-20T22:14:21.225 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.31152.log.gz 2026-02-20T22:14:21.225 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46988.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50198.log 2026-02-20T22:14:21.225 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46988.log.gz 2026-02-20T22:14:21.226 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41800.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31174.log 2026-02-20T22:14:21.226 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50198.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41800.log.gz 2026-02-20T22:14:21.226 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16050.log 2026-02-20T22:14:21.226 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50198.log.gz 2026-02-20T22:14:21.226 INFO:teuthology.orchestra.run.trial107.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.3.47299.log.gz 2026-02-20T22:14:21.226 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31174.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53925.log 2026-02-20T22:14:21.227 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.16050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21845.log 2026-02-20T22:14:21.227 INFO:teuthology.orchestra.run.trial107.stderr: 26.2% -- replaced with /var/log/ceph/cluster2-client.admin.16050.log.gz 2026-02-20T22:14:21.227 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53925.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52602.log 2026-02-20T22:14:21.228 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- 87.7% /var/log/ceph/cluster1-client.mirror.43867.log -- replaced with /var/log/ceph/cluster1-client.mirror.53925.log.gz 2026-02-20T22:14:21.228 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21845.log: 2026-02-20T22:14:21.228 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52602.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52602.log.gz 2026-02-20T22:14:21.228 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22008.log 2026-02-20T22:14:21.228 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43867.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32768.log 2026-02-20T22:14:21.229 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22008.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.21845.log.gz 2026-02-20T22:14:21.229 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31617.log 2026-02-20T22:14:21.229 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32768.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28569.log 2026-02-20T22:14:21.229 INFO:teuthology.orchestra.run.trial107.stderr: 2.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32768.log.gz 2026-02-20T22:14:21.230 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31617.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21301.log 2026-02-20T22:14:21.230 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43867.log.gz 2026-02-20T22:14:21.230 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28569.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-20T22:14:21.230 INFO:teuthology.orchestra.run.trial107.stderr: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28569.log.gz 2026-02-20T22:14:21.230 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21301.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31617.log.gz 2026-02-20T22:14:21.230 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15914.log 2026-02-20T22:14:21.231 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32409.log 2026-02-20T22:14:21.231 INFO:teuthology.orchestra.run.trial107.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.21301.log.gz 2026-02-20T22:14:21.231 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35795.log 2026-02-20T22:14:21.231 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.15914.log: /var/log/ceph/cluster2-client.mirror.32409.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15914.log.gz 2026-02-20T22:14:21.232 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47699.log 2026-02-20T22:14:21.232 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38256.log 2026-02-20T22:14:21.232 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.22008.log.gz 2026-02-20T22:14:21.232 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35795.log.gz 2026-02-20T22:14:21.232 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47699.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40860.log 2026-02-20T22:14:21.233 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47699.log.gz 2026-02-20T22:14:21.233 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38256.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39271.log 2026-02-20T22:14:21.233 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24732.log 2026-02-20T22:14:21.233 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40860.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38256.log.gz 2026-02-20T22:14:21.233 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39271.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29688.log 2026-02-20T22:14:21.233 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24732.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.40860.log.gz 2026-02-20T22:14:21.234 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33289.log 2026-02-20T22:14:21.234 INFO:teuthology.orchestra.run.trial107.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.24732.log.gz 2026-02-20T22:14:21.234 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose 89.8% -- /var/log/ceph/cluster1-client.mirror.26807.log 2026-02-20T22:14:21.234 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.39271.log.gz 2026-02-20T22:14:21.234 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29688.log: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.29688.log.gz 2026-02-20T22:14:21.234 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33289.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55136.log 2026-02-20T22:14:21.235 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26807.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47389.log 2026-02-20T22:14:21.235 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.33289.log.gz 2026-02-20T22:14:21.235 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55136.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54480.log 2026-02-20T22:14:21.235 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55136.log.gz 2026-02-20T22:14:21.236 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47389.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28964.log 2026-02-20T22:14:21.236 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54480.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53166.log 2026-02-20T22:14:21.236 INFO:teuthology.orchestra.run.trial107.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47389.log.gz 2026-02-20T22:14:21.236 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28964.log: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28964.log.gz 2026-02-20T22:14:21.236 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29625.log 2026-02-20T22:14:21.237 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39439.log 2026-02-20T22:14:21.237 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53166.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54480.log.gz 2026-02-20T22:14:21.237 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46822.log 2026-02-20T22:14:21.237 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29625.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53166.log.gz 2026-02-20T22:14:21.237 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39439.log: 77.9%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.39439.log.gz -5 2026-02-20T22:14:21.237 INFO:teuthology.orchestra.run.trial107.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.54671.log 2026-02-20T22:14:21.237 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29625.log.gz 2026-02-20T22:14:21.238 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25328.log 2026-02-20T22:14:21.238 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46822.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26807.log.gz 2026-02-20T22:14:21.238 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54671.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49412.log 2026-02-20T22:14:21.238 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.54671.log.gz 2026-02-20T22:14:21.238 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25328.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52369.log 2026-02-20T22:14:21.239 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49412.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51571.log 2026-02-20T22:14:21.239 INFO:teuthology.orchestra.run.trial107.stderr: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49412.log.gz 2026-02-20T22:14:21.239 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52369.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22606.log 2026-02-20T22:14:21.239 INFO:teuthology.orchestra.run.trial107.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.52369.log.gz 2026-02-20T22:14:21.239 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19971.log 2026-02-20T22:14:21.239 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51571.log.gz 2026-02-20T22:14:21.239 INFO:teuthology.orchestra.run.trial107.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46822.log.gz 2026-02-20T22:14:21.240 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22606.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52240.log 2026-02-20T22:14:21.240 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.19971.log: 4.7%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.19971.log.gz 2026-02-20T22:14:21.240 INFO:teuthology.orchestra.run.trial107.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35638.log 2026-02-20T22:14:21.240 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52240.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53439.log 2026-02-20T22:14:21.240 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35638.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44236.log 2026-02-20T22:14:21.241 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52240.log.gz 2026-02-20T22:14:21.241 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35638.log.gz 2026-02-20T22:14:21.241 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53439.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19988.log 2026-02-20T22:14:21.241 INFO:teuthology.orchestra.run.trial107.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53439.log.gz 2026-02-20T22:14:21.241 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30717.log 2026-02-20T22:14:21.241 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44236.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44236.log.gz 2026-02-20T22:14:21.241 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34072.log 2026-02-20T22:14:21.242 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.19988.log: /var/log/ceph/cluster2-client.mirror.30717.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.19988.log.gz 2026-02-20T22:14:21.242 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40738.log 2026-02-20T22:14:21.242 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30717.log.gz 2026-02-20T22:14:21.242 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34072.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31115.log 2026-02-20T22:14:21.242 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34072.log.gz 2026-02-20T22:14:21.242 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41824.log 2026-02-20T22:14:21.243 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31115.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31115.log.gz 2026-02-20T22:14:21.243 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34671.log 2026-02-20T22:14:21.243 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48993.log 2026-02-20T22:14:21.243 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40738.log.gz 2026-02-20T22:14:21.243 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.41824.log.gz 2026-02-20T22:14:21.244 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34671.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47451.log 2026-02-20T22:14:21.244 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54817.log 2026-02-20T22:14:21.244 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47451.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45272.log 2026-02-20T22:14:21.245 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.54817.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.54817.log.gz 2026-02-20T22:14:21.245 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47174.log 2026-02-20T22:14:21.245 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47451.log.gz 2026-02-20T22:14:21.245 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45272.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52117.log 2026-02-20T22:14:21.245 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45272.log.gz 2026-02-20T22:14:21.245 INFO:teuthology.orchestra.run.trial107.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34671.log.gz 2026-02-20T22:14:21.246 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47174.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48993.log.gz 2026-02-20T22:14:21.246 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49159.log 2026-02-20T22:14:21.246 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47174.log.gz 2026-02-20T22:14:21.246 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52117.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36756.log 2026-02-20T22:14:21.246 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49159.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.52117.log.gz 2026-02-20T22:14:21.246 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29848.log 2026-02-20T22:14:21.246 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49159.log.gz 2026-02-20T22:14:21.247 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36756.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33108.log 2026-02-20T22:14:21.247 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30177.log 2026-02-20T22:14:21.247 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29848.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.36756.log.gz 2026-02-20T22:14:21.247 INFO:teuthology.orchestra.run.trial107.stderr: 86.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29848.log.gz 2026-02-20T22:14:21.247 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33108.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55238.log 2026-02-20T22:14:21.247 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33108.log.gz 2026-02-20T22:14:21.248 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35197.log 2026-02-20T22:14:21.248 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30177.log: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30177.log.gz 2026-02-20T22:14:21.248 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55238.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55238.log.gz 2026-02-20T22:14:21.248 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28764.log 2026-02-20T22:14:21.248 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35197.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40306.log 2026-02-20T22:14:21.248 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28764.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52326.log 2026-02-20T22:14:21.249 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.28764.log.gz 2026-02-20T22:14:21.249 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40306.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21631.log 2026-02-20T22:14:21.249 INFO:teuthology.orchestra.run.trial107.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.35197.log.gz 2026-02-20T22:14:21.249 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.40306.log.gz 2026-02-20T22:14:21.249 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52326.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42855.log 2026-02-20T22:14:21.250 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21631.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47657.log 2026-02-20T22:14:21.250 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52326.log.gz 2026-02-20T22:14:21.250 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41429.log 2026-02-20T22:14:21.250 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42855.log: /var/log/ceph/cluster2-client.mirror.47657.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55110.log 2026-02-20T22:14:21.251 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41429.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34807.log 2026-02-20T22:14:21.251 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55110.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55110.log.gz 2026-02-20T22:14:21.251 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38509.log 2026-02-20T22:14:21.251 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41429.log.gz 2026-02-20T22:14:21.252 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34807.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47132.log 2026-02-20T22:14:21.252 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42855.log.gz 2026-02-20T22:14:21.252 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34807.log.gz 2026-02-20T22:14:21.252 INFO:teuthology.orchestra.run.trial107.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47657.log.gz 2026-02-20T22:14:21.252 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38509.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38097.log 2026-02-20T22:14:21.252 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52008.log 2026-02-20T22:14:21.252 INFO:teuthology.orchestra.run.trial107.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.38509.log.gz 2026-02-20T22:14:21.253 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52303.log 2026-02-20T22:14:21.253 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38097.log: /var/log/ceph/cluster2-client.mirror.52008.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40881.log 2026-02-20T22:14:21.253 INFO:teuthology.orchestra.run.trial107.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38097.log.gz 2026-02-20T22:14:21.253 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52008.log.gz 2026-02-20T22:14:21.253 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35289.log 2026-02-20T22:14:21.253 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52303.log.gz 2026-02-20T22:14:21.254 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40881.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47132.log.gz 2026-02-20T22:14:21.254 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26572.log 2026-02-20T22:14:21.254 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35289.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31746.log 2026-02-20T22:14:21.254 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18939.log 2026-02-20T22:14:21.254 INFO:teuthology.orchestra.run.trial107.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35289.log.gz 2026-02-20T22:14:21.255 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31746.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40881.log.gz 2026-02-20T22:14:21.255 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41619.log 2026-02-20T22:14:21.255 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.31746.log.gz 2026-02-20T22:14:21.255 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18939.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34966.log 2026-02-20T22:14:21.255 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18939.log.gz 2026-02-20T22:14:21.255 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41619.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19237.log 2026-02-20T22:14:21.255 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.41619.log.gz 2026-02-20T22:14:21.256 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30156.log 2026-02-20T22:14:21.256 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34966.log: /var/log/ceph/cluster2-client.admin.19237.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19237.log.gz 2026-02-20T22:14:21.256 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31820.log 2026-02-20T22:14:21.256 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30156.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38300.log 2026-02-20T22:14:21.256 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34966.log.gz 2026-02-20T22:14:21.257 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31820.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29408.log 2026-02-20T22:14:21.257 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31820.log.gz/var/log/ceph/cluster2-client.mirror.38300.log: 2026-02-20T22:14:21.257 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45751.log 2026-02-20T22:14:21.257 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26572.log.gz 2026-02-20T22:14:21.258 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29408.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25581.log 2026-02-20T22:14:21.258 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45751.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29408.log.gz 2026-02-20T22:14:21.258 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43933.log 2026-02-20T22:14:21.258 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38300.log.gz 2026-02-20T22:14:21.258 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25581.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27371.log 2026-02-20T22:14:21.258 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45751.log.gz 2026-02-20T22:14:21.258 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25581.log.gz 2026-02-20T22:14:21.259 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30927.log 2026-02-20T22:14:21.259 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55188.log 2026-02-20T22:14:21.259 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27371.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.43933.log.gz 2026-02-20T22:14:21.259 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25128.log 2026-02-20T22:14:21.259 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43706.log 2026-02-20T22:14:21.259 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55188.log.gz 2026-02-20T22:14:21.259 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27371.log.gz 2026-02-20T22:14:21.260 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30927.log.gz 2026-02-20T22:14:21.260 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25128.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33843.log 2026-02-20T22:14:21.260 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43706.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36734.log 2026-02-20T22:14:21.260 INFO:teuthology.orchestra.run.trial107.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.25128.log.gz 2026-02-20T22:14:21.260 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33843.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43706.log.gz 2026-02-20T22:14:21.260 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40631.log 2026-02-20T22:14:21.260 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33843.log.gz 2026-02-20T22:14:21.260 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50783.log 2026-02-20T22:14:21.261 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40631.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36645.log 2026-02-20T22:14:21.261 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40631.log.gz 2026-02-20T22:14:21.261 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50783.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44814.log 2026-02-20T22:14:21.261 INFO:teuthology.orchestra.run.trial107.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.36734.log.gz 2026-02-20T22:14:21.261 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36645.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50783.log.gz 2026-02-20T22:14:21.261 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33263.log 2026-02-20T22:14:21.261 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36645.log.gz 2026-02-20T22:14:21.262 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44814.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19954.log 2026-02-20T22:14:21.262 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44814.log.gz 2026-02-20T22:14:21.262 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.33263.log: /var/log/ceph/cluster1-client.mirror.50513.log 2026-02-20T22:14:21.262 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.19954.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34096.log 2026-02-20T22:14:21.262 INFO:teuthology.orchestra.run.trial107.stderr: 63.2% -- replaced with /var/log/ceph/cluster2-client.mirror.19954.log.gz 2026-02-20T22:14:21.263 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50513.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50513.log.gz 2026-02-20T22:14:21.263 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37449.log 2026-02-20T22:14:21.263 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34096.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.33263.log.gz 2026-02-20T22:14:21.263 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29955.log 2026-02-20T22:14:21.263 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34096.log.gz 2026-02-20T22:14:21.263 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37449.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27753.log 2026-02-20T22:14:21.264 INFO:teuthology.orchestra.run.trial107.stderr:gzip/var/log/ceph/cluster2-client.mirror.29955.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54348.log 2026-02-20T22:14:21.264 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27753.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24660.log 2026-02-20T22:14:21.264 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27753.log.gz 2026-02-20T22:14:21.264 INFO:teuthology.orchestra.run.trial107.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37449.log.gz 2026-02-20T22:14:21.264 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54348.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44971.log 2026-02-20T22:14:21.265 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54348.log.gz 2026-02-20T22:14:21.265 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29955.log.gz 2026-02-20T22:14:21.265 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43754.log 2026-02-20T22:14:21.265 INFO:teuthology.orchestra.run.trial107.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.24660.log.gz 2026-02-20T22:14:21.265 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44971.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49523.log 2026-02-20T22:14:21.265 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44971.log.gz 2026-02-20T22:14:21.265 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43754.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.48303.log 2026-02-20T22:14:21.266 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49523.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.43754.log.gz 2026-02-20T22:14:21.266 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26529.log 2026-02-20T22:14:21.266 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49523.log.gz 2026-02-20T22:14:21.266 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.1.48303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54221.log 2026-02-20T22:14:21.266 INFO:teuthology.orchestra.run.trial107.stderr:gzip/var/log/ceph/cluster2-client.mirror.26529.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54327.log 2026-02-20T22:14:21.267 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48476.log 2026-02-20T22:14:21.267 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54327.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44692.log 2026-02-20T22:14:21.267 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48476.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54221.log.gz 2026-02-20T22:14:21.268 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25152.log 2026-02-20T22:14:21.268 INFO:teuthology.orchestra.run.trial107.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48476.log.gz 2026-02-20T22:14:21.268 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44692.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51720.log 2026-02-20T22:14:21.268 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18620.log 2026-02-20T22:14:21.268 INFO:teuthology.orchestra.run.trial107.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.25152.log.gz 2026-02-20T22:14:21.268 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51720.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.44692.log.gz 2026-02-20T22:14:21.269 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51720.log.gz 2026-02-20T22:14:21.269 INFO:teuthology.orchestra.run.trial107.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54327.log.gz 2026-02-20T22:14:21.269 INFO:teuthology.orchestra.run.trial107.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.25221.log 2026-02-20T22:14:21.269 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18620.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18620.log.gz 2026-02-20T22:14:21.269 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35450.log 2026-02-20T22:14:21.269 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52866.log 2026-02-20T22:14:21.270 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35450.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30837.log 2026-02-20T22:14:21.270 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.48303.log.gz 2026-02-20T22:14:21.270 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37404.log 2026-02-20T22:14:21.270 INFO:teuthology.orchestra.run.trial107.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52866.log.gz 2026-02-20T22:14:21.270 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30837.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33659.log 2026-02-20T22:14:21.271 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37404.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33337.log 2026-02-20T22:14:21.271 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30837.log.gz 2026-02-20T22:14:21.271 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26529.log.gz 2026-02-20T22:14:21.271 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.35450.log.gz 2026-02-20T22:14:21.271 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37404.log.gz 2026-02-20T22:14:21.271 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33659.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.31398.log 2026-02-20T22:14:21.271 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33337.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48538.log 2026-02-20T22:14:21.271 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33337.log.gz 2026-02-20T22:14:21.271 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.0.31398.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19714.log 2026-02-20T22:14:21.272 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48538.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43549.log 2026-02-20T22:14:21.272 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33659.log.gz 2026-02-20T22:14:21.272 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48538.log.gz 2026-02-20T22:14:21.272 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19714.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.19714.log.gz 2026-02-20T22:14:21.272 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37916.log 2026-02-20T22:14:21.272 INFO:teuthology.orchestra.run.trial107.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25221.log.gz 2026-02-20T22:14:21.272 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26721.log 2026-02-20T22:14:21.273 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43549.log.gz 2026-02-20T22:14:21.273 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.23297.log 2026-02-20T22:14:21.273 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37916.log: /var/log/ceph/cluster1-client.mirror.26721.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25698.log 2026-02-20T22:14:21.273 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37916.log.gz 2026-02-20T22:14:21.273 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.0.23297.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45542.log 2026-02-20T22:14:21.274 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25698.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43594.log 2026-02-20T22:14:21.274 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45542.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33406.log 2026-02-20T22:14:21.275 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43594.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38677.log 2026-02-20T22:14:21.275 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33406.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.0.31398.log.gz 2026-02-20T22:14:21.275 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45542.log.gz 2026-02-20T22:14:21.275 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45962.log 2026-02-20T22:14:21.275 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38677.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.0.23297.log.gz 2026-02-20T22:14:21.276 INFO:teuthology.orchestra.run.trial107.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43594.log.gz 2026-02-20T22:14:21.276 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38677.log.gz 2026-02-20T22:14:21.276 INFO:teuthology.orchestra.run.trial107.stderr: -- /var/log/ceph/cluster2-client.mirror.39017.log 2026-02-20T22:14:21.276 INFO:teuthology.orchestra.run.trial107.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.33406.log.gz 2026-02-20T22:14:21.276 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45962.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26721.log.gz 2026-02-20T22:14:21.276 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52161.log 2026-02-20T22:14:21.276 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36963.log 2026-02-20T22:14:21.277 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52161.log: 93.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51930.log 2026-02-20T22:14:21.277 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.25698.log.gz 2026-02-20T22:14:21.277 INFO:teuthology.orchestra.run.trial107.stderr: 90.0% 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.39017.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.52161.log.gz 2026-02-20T22:14:21.277 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:21.277 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48414.log 2026-02-20T22:14:21.277 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51930.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.45962.log.gz 2026-02-20T22:14:21.278 INFO:teuthology.orchestra.run.trial107.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.51930.log.gz 2026-02-20T22:14:21.278 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19036.log 2026-02-20T22:14:21.278 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48414.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51008.log 2026-02-20T22:14:21.278 INFO:teuthology.orchestra.run.trial107.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.36963.log.gz 2026-02-20T22:14:21.278 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19036.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48414.log.gz 2026-02-20T22:14:21.278 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32703.log 2026-02-20T22:14:21.278 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19036.log.gz 2026-02-20T22:14:21.279 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51008.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51685.log 2026-02-20T22:14:21.279 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51008.log.gz 2026-02-20T22:14:21.279 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32703.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22274.log 2026-02-20T22:14:21.279 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51685.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34903.log 2026-02-20T22:14:21.279 INFO:teuthology.orchestra.run.trial107.stderr: 42.2% -- replaced with /var/log/ceph/cluster1-client.mirror.51685.log.gz 2026-02-20T22:14:21.279 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.22274.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32703.log.gz 2026-02-20T22:14:21.279 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43427.log 2026-02-20T22:14:21.280 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.22274.log.gz 2026-02-20T22:14:21.280 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34903.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54286.log 2026-02-20T22:14:21.280 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34903.log.gz 2026-02-20T22:14:21.280 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25967.log 2026-02-20T22:14:21.280 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43427.log: /var/log/ceph/cluster2-client.mirror.54286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42835.log 2026-02-20T22:14:21.280 INFO:teuthology.orchestra.run.trial107.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54286.log.gz 2026-02-20T22:14:21.281 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28179.log 2026-02-20T22:14:21.281 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25967.log: /var/log/ceph/cluster2-client.mirror.42835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32350.log 2026-02-20T22:14:21.281 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43427.log.gz 2026-02-20T22:14:21.281 INFO:teuthology.orchestra.run.trial107.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.25967.log.gz 2026-02-20T22:14:21.281 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28179.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31997.log 2026-02-20T22:14:21.282 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32350.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25895.log 2026-02-20T22:14:21.282 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32350.log.gz 2026-02-20T22:14:21.282 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38906.log 2026-02-20T22:14:21.282 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28179.log.gz 2026-02-20T22:14:21.282 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31997.log: /var/log/ceph/cluster2-client.mirror.25895.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42835.log.gz 2026-02-20T22:14:21.282 INFO:teuthology.orchestra.run.trial107.stderr: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31997.log.gz 2026-02-20T22:14:21.282 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44212.log 2026-02-20T22:14:21.282 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38906.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37657.log 2026-02-20T22:14:21.283 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.38906.log.gz 2026-02-20T22:14:21.283 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44212.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48434.log 2026-02-20T22:14:21.283 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44212.log.gz 2026-02-20T22:14:21.283 INFO:teuthology.orchestra.run.trial107.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25895.log.gz 2026-02-20T22:14:21.283 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37657.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37707.log 2026-02-20T22:14:21.283 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31213.log 2026-02-20T22:14:21.283 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37657.log.gz 2026-02-20T22:14:21.284 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34578.log 2026-02-20T22:14:21.284 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18524.log 2026-02-20T22:14:21.284 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34578.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21587.log 2026-02-20T22:14:21.284 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31213.log.gz 2026-02-20T22:14:21.285 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37707.log.gz 2026-02-20T22:14:21.285 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.34578.log.gz 2026-02-20T22:14:21.285 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28063.log 2026-02-20T22:14:21.285 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18524.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18524.log.gz 2026-02-20T22:14:21.285 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.21587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55058.log 2026-02-20T22:14:21.285 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28063.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27611.log 2026-02-20T22:14:21.285 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55058.log: gzip 90.5% -5 --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.48434.log.gz /var/log/ceph/cluster2-client.mirror.48952.log 2026-02-20T22:14:21.286 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:21.286 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55058.log.gz 2026-02-20T22:14:21.286 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27611.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48177.log 2026-02-20T22:14:21.286 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27611.log.gz 2026-02-20T22:14:21.286 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48952.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31510.log 2026-02-20T22:14:21.286 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32966.log 2026-02-20T22:14:21.287 INFO:teuthology.orchestra.run.trial107.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48952.log.gz 2026-02-20T22:14:21.287 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31510.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50086.log 2026-02-20T22:14:21.287 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48177.log.gz 2026-02-20T22:14:21.287 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32966.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29747.log 2026-02-20T22:14:21.287 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50086.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31510.log.gz 2026-02-20T22:14:21.287 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36874.log 2026-02-20T22:14:21.287 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50086.log.gz 2026-02-20T22:14:21.288 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29747.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28507.log 89.8% 2026-02-20T22:14:21.288 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.32966.log.gz 2026-02-20T22:14:21.288 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36874.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39643.log 2026-02-20T22:14:21.288 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36874.log.gz 2026-02-20T22:14:21.288 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28507.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45476.log 2026-02-20T22:14:21.288 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.21587.log.gz 2026-02-20T22:14:21.289 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39643.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34944.log 2026-02-20T22:14:21.289 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28507.log.gz 2026-02-20T22:14:21.289 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45476.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40330.log 2026-02-20T22:14:21.289 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.39643.log.gz 2026-02-20T22:14:21.289 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34944.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30466.log 2026-02-20T22:14:21.290 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40330.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31709.log 2026-02-20T22:14:21.290 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40330.log.gz 2026-02-20T22:14:21.290 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.45476.log.gz 2026-02-20T22:14:21.290 INFO:teuthology.orchestra.run.trial107.stderr:gzip/var/log/ceph/cluster2-client.mirror.30466.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47719.log 2026-02-20T22:14:21.290 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.34944.log.gz 2026-02-20T22:14:21.290 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31709.log: gzip 87.7% -5 --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.30466.log.gz /var/log/ceph/cluster1-client.mirror.50401.log 2026-02-20T22:14:21.290 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:21.291 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31709.log.gz 2026-02-20T22:14:21.291 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47719.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38882.log 2026-02-20T22:14:21.291 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50401.log: gzip -5 --verbose 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50401.log.gz 2026-02-20T22:14:21.291 INFO:teuthology.orchestra.run.trial107.stderr: -- /var/log/ceph/cluster1-client.mirror.34879.log 2026-02-20T22:14:21.292 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38882.log: gzip -5 --verbose 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38882.log.gz 2026-02-20T22:14:21.292 INFO:teuthology.orchestra.run.trial107.stderr: -- /var/log/ceph/cluster2-client.mirror.44395.log 2026-02-20T22:14:21.292 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34879.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44766.log 87.8% 2026-02-20T22:14:21.292 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.34879.log.gz 2026-02-20T22:14:21.292 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44395.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47719.log.gz 2026-02-20T22:14:21.292 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51391.log 2026-02-20T22:14:21.292 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44395.log.gz 2026-02-20T22:14:21.293 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44766.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54841.log 2026-02-20T22:14:21.293 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44766.log.gz 2026-02-20T22:14:21.293 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33590.log 2026-02-20T22:14:21.293 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51391.log: 84.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51391.log.gz 2026-02-20T22:14:21.293 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.54841.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31493.log 2026-02-20T22:14:21.293 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.54841.log.gz 2026-02-20T22:14:21.294 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49568.log 2026-02-20T22:14:21.294 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33590.log: /var/log/ceph/cluster2-client.mirror.31493.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31493.log.gz 2026-02-20T22:14:21.294 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33590.log.gz 2026-02-20T22:14:21.294 INFO:teuthology.orchestra.run.trial107.stderr: -- /var/log/ceph/cluster2-client.mirror.2.53582.log 2026-02-20T22:14:21.294 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49568.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49568.log.gz 2026-02-20T22:14:21.294 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33867.log 2026-02-20T22:14:21.295 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.2.53582.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40378.log 2026-02-20T22:14:21.295 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33867.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44537.log 2026-02-20T22:14:21.295 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33867.log.gz 2026-02-20T22:14:21.295 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40378.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50738.log 2026-02-20T22:14:21.295 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.40378.log.gz 2026-02-20T22:14:21.296 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24636.log 2026-02-20T22:14:21.296 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44537.log: /var/log/ceph/cluster1-client.mirror.50738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46409.log 2026-02-20T22:14:21.296 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44537.log.gz 2026-02-20T22:14:21.296 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50738.log.gz 2026-02-20T22:14:21.296 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24636.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42814.log 2026-02-20T22:14:21.296 INFO:teuthology.orchestra.run.trial107.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.24636.log.gz 2026-02-20T22:14:21.297 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28707.log 2026-02-20T22:14:21.297 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24182.log/var/log/ceph/cluster1-client.mirror.42814.log: 2026-02-20T22:14:21.297 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46409.log.gz 2026-02-20T22:14:21.297 INFO:teuthology.orchestra.run.trial107.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.2.53582.log.gz 2026-02-20T22:14:21.297 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40672.log 2026-02-20T22:14:21.297 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42814.log.gz 2026-02-20T22:14:21.298 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24182.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29608.log 2026-02-20T22:14:21.298 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40672.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.24182.log.gz 2026-02-20T22:14:21.298 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29608.log: 80.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29608.log.gz 2026-02-20T22:14:21.298 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24797.log 2026-02-20T22:14:21.299 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39062.log 2026-02-20T22:14:21.299 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.40672.log.gz 2026-02-20T22:14:21.299 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32056.log 2026-02-20T22:14:21.300 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39062.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53308.log 2026-02-20T22:14:21.300 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32056.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.39062.log.gz 2026-02-20T22:14:21.300 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27308.log 2026-02-20T22:14:21.300 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53308.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35361.log 2026-02-20T22:14:21.301 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27308.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38003.log 2026-02-20T22:14:21.301 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35361.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24254.log 2026-02-20T22:14:21.302 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35361.log.gz 2026-02-20T22:14:21.302 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38003.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43361.log 2026-02-20T22:14:21.302 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24254.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.24797.log.gz 2026-02-20T22:14:21.302 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38003.log.gz 2026-02-20T22:14:21.303 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.24254.log.gz 2026-02-20T22:14:21.303 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37428.log 2026-02-20T22:14:21.303 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40812.log 2026-02-20T22:14:21.303 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43361.log: /var/log/ceph/cluster1-client.mirror.37428.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42350.log 2026-02-20T22:14:21.304 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37428.log.gz/var/log/ceph/cluster1-client.mirror.40812.log: 2026-02-20T22:14:21.304 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.40812.log.gz 2026-02-20T22:14:21.304 INFO:teuthology.orchestra.run.trial107.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.43361.log.gz 2026-02-20T22:14:21.304 INFO:teuthology.orchestra.run.trial107.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27308.log.gz 2026-02-20T22:14:21.305 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44626.log 2026-02-20T22:14:21.305 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54865.log 2026-02-20T22:14:21.305 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42350.log: /var/log/ceph/cluster1-client.mirror.44626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40149.log 2026-02-20T22:14:21.305 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42350.log.gz 2026-02-20T22:14:21.306 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.54865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44284.log 2026-02-20T22:14:21.306 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.54865.log.gz 2026-02-20T22:14:21.306 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40149.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52924.log 2026-02-20T22:14:21.306 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19435.log 2026-02-20T22:14:21.306 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44626.log.gz 2026-02-20T22:14:21.306 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40149.log.gz/var/log/ceph/cluster1-client.mirror.44284.log: 2026-02-20T22:14:21.306 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52924.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44284.log.gz 2026-02-20T22:14:21.307 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37175.log 2026-02-20T22:14:21.307 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52924.log.gz 2026-02-20T22:14:21.307 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.32085.log 2026-02-20T22:14:21.307 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19435.log: /var/log/ceph/cluster1-client.mirror.37175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46719.log 2026-02-20T22:14:21.308 INFO:teuthology.orchestra.run.trial107.stderr: 91.0% -- replaced with /var/log/ceph/cluster1-client.mirror.19435.log.gz 2026-02-20T22:14:21.308 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.0.32085.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37175.log.gz 2026-02-20T22:14:21.308 INFO:teuthology.orchestra.run.trial032.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-20T22:14:21.308 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52282.log 2026-02-20T22:14:21.308 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46719.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40836.log 2026-02-20T22:14:21.308 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52282.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23249.log 2026-02-20T22:14:21.309 INFO:teuthology.orchestra.run.trial107.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46719.log.gz 2026-02-20T22:14:21.309 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40836.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40836.log.gz 2026-02-20T22:14:21.309 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46091.log 2026-02-20T22:14:21.309 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.23249.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31315.log 2026-02-20T22:14:21.310 INFO:teuthology.orchestra.run.trial107.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.23249.log.gz 2026-02-20T22:14:21.310 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46091.log: 88.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54113.log 2026-02-20T22:14:21.310 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46091.log.gz 2026-02-20T22:14:21.310 INFO:teuthology.orchestra.run.trial107.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.0.32085.log.gz 2026-02-20T22:14:21.311 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31315.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50131.log 2026-02-20T22:14:21.311 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54034.log 2026-02-20T22:14:21.311 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31315.log.gz/var/log/ceph/cluster2-client.mirror.54113.log: 2026-02-20T22:14:21.311 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50131.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50131.log.gz 2026-02-20T22:14:21.311 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43983.log 2026-02-20T22:14:21.312 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54113.log.gz 2026-02-20T22:14:21.312 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54034.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45795.log 2026-02-20T22:14:21.312 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54034.log.gz/var/log/ceph/cluster1-client.mirror.43983.log: 2026-02-20T22:14:21.312 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43983.log.gz 2026-02-20T22:14:21.313 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43959.log 2026-02-20T22:14:21.313 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45795.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33932.log 2026-02-20T22:14:21.313 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43959.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44007.log 2026-02-20T22:14:21.313 INFO:teuthology.orchestra.run.trial107.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.45795.log.gz 2026-02-20T22:14:21.314 INFO:teuthology.orchestra.run.trial107.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.43959.log.gz 2026-02-20T22:14:21.315 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33932.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.33932.log.gz 2026-02-20T22:14:21.315 INFO:teuthology.orchestra.run.trial032.stderr: 91.2% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-20T22:14:21.315 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27002.log 2026-02-20T22:14:21.315 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-20T22:14:21.315 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44007.log: /var/log/ceph/cluster2-client.mirror.27002.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21498.log 2026-02-20T22:14:21.316 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44007.log.gz 2026-02-20T22:14:21.316 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34602.log 2026-02-20T22:14:21.316 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.21498.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.21498.log.gz 2026-02-20T22:14:21.317 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34207.log 2026-02-20T22:14:21.317 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34602.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34602.log.gz 2026-02-20T22:14:21.318 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44648.log 2026-02-20T22:14:21.318 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34207.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.34207.log.gz 2026-02-20T22:14:21.319 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36714.log 2026-02-20T22:14:21.319 INFO:teuthology.orchestra.run.trial107.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27002.log.gz 2026-02-20T22:14:21.320 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44648.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44648.log.gz 2026-02-20T22:14:21.320 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45296.log 2026-02-20T22:14:21.320 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28083.log 2026-02-20T22:14:21.321 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36714.log: /var/log/ceph/cluster1-client.mirror.45296.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45296.log.gz 2026-02-20T22:14:21.321 INFO:teuthology.orchestra.run.trial107.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.36714.log.gz 2026-02-20T22:14:21.322 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42995.log 2026-02-20T22:14:21.322 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55162.log 2026-02-20T22:14:21.322 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28083.log: /var/log/ceph/cluster1-client.mirror.42995.log: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28083.log.gz 2026-02-20T22:14:21.322 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42995.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50986.log 2026-02-20T22:14:21.323 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:21.323 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55162.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55162.log.gz 2026-02-20T22:14:21.323 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48091.log 2026-02-20T22:14:21.324 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19646.log 2026-02-20T22:14:21.324 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50986.log: /var/log/ceph/cluster2-client.mirror.48091.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50986.log.gz 2026-02-20T22:14:21.324 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45154.log 2026-02-20T22:14:21.324 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19646.log: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.19646.log.gz 2026-02-20T22:14:21.325 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44353.log 2026-02-20T22:14:21.325 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45154.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48091.log.gz 2026-02-20T22:14:21.325 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45154.log.gz 2026-02-20T22:14:21.326 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26764.log 2026-02-20T22:14:21.326 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29529.log 2026-02-20T22:14:21.327 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44353.log: /var/log/ceph/cluster1-client.mirror.26764.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44353.log.gz 2026-02-20T22:14:21.328 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35263.log 2026-02-20T22:14:21.328 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29529.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29529.log.gz 2026-02-20T22:14:21.328 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27328.log 2026-02-20T22:14:21.329 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35263.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35263.log.gz 2026-02-20T22:14:21.329 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24035.log 2026-02-20T22:14:21.330 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.27328.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.26764.log.gz 2026-02-20T22:14:21.333 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.27328.log.gz 2026-02-20T22:14:21.333 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32261.log 2026-02-20T22:14:21.334 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50873.log 2026-02-20T22:14:21.334 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.24035.log: /var/log/ceph/cluster1-client.mirror.32261.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.24035.log.gz 2026-02-20T22:14:21.334 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23648.log 2026-02-20T22:14:21.335 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50873.log: 85.2% -- replaced with /var/log/ceph/cluster1-client.mirror.50873.log.gz 2026-02-20T22:14:21.335 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44031.log 2026-02-20T22:14:21.339 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23648.log: 95.5% -- replaced with /var/log/ceph/cluster1-client.mirror.32261.log.gz 2026-02-20T22:14:21.345 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54771.log 2026-02-20T22:14:21.346 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44031.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44031.log.gz 2026-02-20T22:14:21.346 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28428.log 2026-02-20T22:14:21.347 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54771.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54771.log.gz 2026-02-20T22:14:21.347 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45664.log 2026-02-20T22:14:21.348 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28428.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28428.log.gz 2026-02-20T22:14:21.348 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38350.log 2026-02-20T22:14:21.349 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45664.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.45664.log.gz 2026-02-20T22:14:21.349 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43296.log 2026-02-20T22:14:21.350 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38350.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38350.log.gz 2026-02-20T22:14:21.350 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48050.log 2026-02-20T22:14:21.350 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43296.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43296.log.gz 2026-02-20T22:14:21.351 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26635.log 2026-02-20T22:14:21.352 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48050.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48050.log.gz 2026-02-20T22:14:21.352 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28665.log 2026-02-20T22:14:21.356 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26635.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26635.log.gz 2026-02-20T22:14:21.356 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52624.log 2026-02-20T22:14:21.357 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28665.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28665.log.gz 2026-02-20T22:14:21.357 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50468.log 2026-02-20T22:14:21.365 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52624.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49221.log 2026-02-20T22:14:21.366 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50468.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50468.log.gz 2026-02-20T22:14:21.374 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48261.log 2026-02-20T22:14:21.374 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49221.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49221.log.gz 2026-02-20T22:14:21.389 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48071.log 2026-02-20T22:14:21.390 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48261.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48261.log.gz 2026-02-20T22:14:21.397 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53266.log 2026-02-20T22:14:21.398 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48071.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48071.log.gz 2026-02-20T22:14:21.405 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47864.log 2026-02-20T22:14:21.406 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53266.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53266.log.gz 2026-02-20T22:14:21.417 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36349.log 2026-02-20T22:14:21.418 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47864.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47864.log.gz 2026-02-20T22:14:21.429 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41893.log 2026-02-20T22:14:21.430 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36349.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36349.log.gz 2026-02-20T22:14:21.441 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33084.log 2026-02-20T22:14:21.443 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41893.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41893.log.gz 2026-02-20T22:14:21.454 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42422.log 2026-02-20T22:14:21.454 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33084.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33084.log.gz 2026-02-20T22:14:21.474 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.29124.log 2026-02-20T22:14:21.475 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42422.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42422.log.gz 2026-02-20T22:14:21.485 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28606.log 2026-02-20T22:14:21.505 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.1.29124.log: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.29124.log.gz 2026-02-20T22:14:21.509 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39800.log 2026-02-20T22:14:21.510 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28606.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.28606.log.gz 2026-02-20T22:14:21.523 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25361.log 2026-02-20T22:14:21.523 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39800.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39800.log.gz 2026-02-20T22:14:21.535 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35867.log 2026-02-20T22:14:21.535 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.25361.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.25361.log.gz 2026-02-20T22:14:21.547 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19799.log 2026-02-20T22:14:21.547 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35867.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.35867.log.gz 2026-02-20T22:14:21.558 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50693.log 2026-02-20T22:14:21.559 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19799.log: 64.8% -- replaced with /var/log/ceph/cluster1-client.mirror.19799.log.gz 2026-02-20T22:14:21.569 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36826.log 2026-02-20T22:14:21.570 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50693.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50693.log.gz 2026-02-20T22:14:21.582 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54573.log 2026-02-20T22:14:21.582 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36826.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36826.log.gz 2026-02-20T22:14:21.586 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.27982.log.gz 2026-02-20T22:14:21.587 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40256.log 2026-02-20T22:14:21.588 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54573.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54573.log.gz 2026-02-20T22:14:21.588 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53844.log 2026-02-20T22:14:21.589 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40256.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40256.log.gz 2026-02-20T22:14:21.589 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54593.log 2026-02-20T22:14:21.590 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53844.log: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53844.log.gz 2026-02-20T22:14:21.590 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31659.log 2026-02-20T22:14:21.592 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54593.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54593.log.gz 2026-02-20T22:14:21.592 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29102.log 2026-02-20T22:14:21.601 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31659.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.25436.log 2026-02-20T22:14:21.612 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29102.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30156.log.gz 2026-02-20T22:14:21.617 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35516.log 2026-02-20T22:14:21.622 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.0.25436.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.25436.log.gz 2026-02-20T22:14:21.626 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49681.log 2026-02-20T22:14:21.627 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35516.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35516.log.gz 2026-02-20T22:14:21.634 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31174.log.gz 2026-02-20T22:14:21.635 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46153.log 2026-02-20T22:14:21.635 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49681.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49681.log.gz 2026-02-20T22:14:21.636 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27491.log 2026-02-20T22:14:21.637 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46153.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46153.log.gz 2026-02-20T22:14:21.637 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24820.log 2026-02-20T22:14:21.637 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27491.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.27491.log.gz 2026-02-20T22:14:21.638 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44079.log 2026-02-20T22:14:21.642 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24820.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24820.log.gz 2026-02-20T22:14:21.642 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50243.log 2026-02-20T22:14:21.643 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44079.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44079.log.gz 2026-02-20T22:14:21.643 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48156.log 2026-02-20T22:14:21.644 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50243.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50243.log.gz 2026-02-20T22:14:21.644 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25176.log 2026-02-20T22:14:21.646 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48156.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48156.log.gz 2026-02-20T22:14:21.646 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43453.log 2026-02-20T22:14:21.647 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25176.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.25176.log.gz 2026-02-20T22:14:21.647 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49883.log 2026-02-20T22:14:21.648 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43453.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.43453.log.gz 2026-02-20T22:14:21.648 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30114.log 2026-02-20T22:14:21.649 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49883.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49883.log.gz 2026-02-20T22:14:21.649 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28142.log 2026-02-20T22:14:21.650 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30114.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30114.log.gz 2026-02-20T22:14:21.650 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43525.log 2026-02-20T22:14:21.658 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28142.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48281.log 2026-02-20T22:14:21.658 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43525.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43525.log.gz 2026-02-20T22:14:21.672 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32658.log.gz 2026-02-20T22:14:21.673 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21871.log 2026-02-20T22:14:21.675 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48281.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48281.log.gz 2026-02-20T22:14:21.675 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33385.log 2026-02-20T22:14:21.675 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21871.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.21871.log.gz 2026-02-20T22:14:21.676 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44373.log 2026-02-20T22:14:21.676 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33385.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33385.log.gz 2026-02-20T22:14:21.677 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27411.log 2026-02-20T22:14:21.678 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44373.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.44373.log.gz 2026-02-20T22:14:21.678 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33177.log 2026-02-20T22:14:21.679 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27411.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.27411.log.gz 2026-02-20T22:14:21.679 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45917.log 2026-02-20T22:14:21.680 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33177.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33177.log.gz 2026-02-20T22:14:21.681 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29043.log 2026-02-20T22:14:21.681 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45917.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45917.log.gz 2026-02-20T22:14:21.682 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38398.log 2026-02-20T22:14:21.682 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29043.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29043.log.gz 2026-02-20T22:14:21.683 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48682.log 2026-02-20T22:14:21.683 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38398.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.38398.log.gz 2026-02-20T22:14:21.683 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.12134.log 2026-02-20T22:14:21.685 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48682.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48682.log.gz 2026-02-20T22:14:21.686 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46450.log 2026-02-20T22:14:21.686 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/ceph.tmp-client.admin.12134.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.12134.log.gz 2026-02-20T22:14:21.687 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22562.log 2026-02-20T22:14:21.688 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46450.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46450.log.gz 2026-02-20T22:14:21.689 INFO:teuthology.orchestra.run.trial107.stderr: 94.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42668.log 2026-02-20T22:14:21.689 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.25328.log.gz 2026-02-20T22:14:21.689 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22562.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50491.log 2026-02-20T22:14:21.690 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42668.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42668.log.gz 2026-02-20T22:14:21.691 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15938.log 2026-02-20T22:14:21.691 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50491.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50491.log.gz 2026-02-20T22:14:21.692 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53560.log 2026-02-20T22:14:21.692 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.15938.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15938.log.gz 2026-02-20T22:14:21.692 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22949.log 2026-02-20T22:14:21.692 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.22562.log.gz 2026-02-20T22:14:21.693 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53560.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32636.log 2026-02-20T22:14:21.693 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22949.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.22949.log.gz 2026-02-20T22:14:21.703 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32409.log.gz 2026-02-20T22:14:21.707 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43043.log 2026-02-20T22:14:21.707 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29705.log 2026-02-20T22:14:21.707 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32636.log.gz 2026-02-20T22:14:21.708 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43043.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43043.log.gz 2026-02-20T22:14:21.714 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25200.log 2026-02-20T22:14:21.714 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29705.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.52711.log 2026-02-20T22:14:21.715 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25200.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29705.log.gz 2026-02-20T22:14:21.715 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.25200.log.gz 2026-02-20T22:14:21.715 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40559.log 2026-02-20T22:14:21.716 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.0.52711.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45640.log 2026-02-20T22:14:21.716 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40559.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40559.log.gz 2026-02-20T22:14:21.717 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29250.log 2026-02-20T22:14:21.717 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45640.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45640.log.gz 2026-02-20T22:14:21.718 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51323.log 2026-02-20T22:14:21.719 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29250.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29250.log.gz 2026-02-20T22:14:21.719 INFO:teuthology.orchestra.run.trial107.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.52711.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51779.log 2026-02-20T22:14:21.719 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:21.720 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51323.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51323.log.gz 2026-02-20T22:14:21.730 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27081.log 2026-02-20T22:14:21.730 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51779.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49658.log 2026-02-20T22:14:21.734 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.27081.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.27081.log.gz 2026-02-20T22:14:21.735 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19010.log 2026-02-20T22:14:21.735 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49658.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49658.log.gz 2026-02-20T22:14:21.735 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49996.log 2026-02-20T22:14:21.736 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19010.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19010.log.gz 2026-02-20T22:14:21.736 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54928.log 2026-02-20T22:14:21.736 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49996.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49996.log.gz 2026-02-20T22:14:21.737 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38834.log 2026-02-20T22:14:21.738 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.54928.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.54928.log.gz 2026-02-20T22:14:21.738 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37238.log 2026-02-20T22:14:21.739 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38834.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38834.log.gz 2026-02-20T22:14:21.739 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46884.log 2026-02-20T22:14:21.739 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37238.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.37238.log.gz 2026-02-20T22:14:21.740 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48930.log 2026-02-20T22:14:21.741 INFO:teuthology.orchestra.run.trial032.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-20T22:14:21.742 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46884.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46884.log.gz 2026-02-20T22:14:21.742 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35036.log 2026-02-20T22:14:21.743 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48930.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48930.log.gz 2026-02-20T22:14:21.744 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41176.log 2026-02-20T22:14:21.744 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35036.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35036.log.gz 2026-02-20T22:14:21.745 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30604.log 2026-02-20T22:14:21.746 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41176.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.41176.log.gz 2026-02-20T22:14:21.746 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26420.log 2026-02-20T22:14:21.748 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30604.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30604.log.gz 2026-02-20T22:14:21.748 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43067.log 2026-02-20T22:14:21.751 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26420.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26420.log.gz 2026-02-20T22:14:21.751 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32723.log 2026-02-20T22:14:21.752 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43067.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.43067.log.gz 2026-02-20T22:14:21.752 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.32261.log 2026-02-20T22:14:21.753 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32723.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32723.log.gz 2026-02-20T22:14:21.753 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38975.log 2026-02-20T22:14:21.774 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.2.32261.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48806.log 2026-02-20T22:14:21.775 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38975.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38975.log.gz 2026-02-20T22:14:21.782 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52624.log.gz 2026-02-20T22:14:21.783 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35662.log 2026-02-20T22:14:21.785 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48806.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48806.log.gz 2026-02-20T22:14:21.785 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20149.log 2026-02-20T22:14:21.786 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35662.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.35662.log.gz 2026-02-20T22:14:21.786 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49076.log 2026-02-20T22:14:21.787 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.20149.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.20149.log.gz 2026-02-20T22:14:21.788 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49179.log 2026-02-20T22:14:21.788 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49076.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49076.log.gz 2026-02-20T22:14:21.788 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36985.log 2026-02-20T22:14:21.790 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49179.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49179.log.gz 2026-02-20T22:14:21.790 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45616.log 2026-02-20T22:14:21.791 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36985.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36985.log.gz 2026-02-20T22:14:21.792 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30488.log 2026-02-20T22:14:21.792 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45616.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45616.log.gz 2026-02-20T22:14:21.792 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34650.log 2026-02-20T22:14:21.793 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30488.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22606.log.gz 2026-02-20T22:14:21.794 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48641.log 2026-02-20T22:14:21.794 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34650.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34650.log.gz 2026-02-20T22:14:21.795 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30034.log 2026-02-20T22:14:21.795 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48641.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48641.log.gz 2026-02-20T22:14:21.796 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29509.log 2026-02-20T22:14:21.797 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30034.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.30034.log.gz 2026-02-20T22:14:21.797 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33060.log 2026-02-20T22:14:21.801 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29509.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28865.log.gz 2026-02-20T22:14:21.804 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28707.log.gz 2026-02-20T22:14:21.804 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29667.log.gz 2026-02-20T22:14:21.805 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23584.log 2026-02-20T22:14:21.807 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33060.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51458.log 2026-02-20T22:14:21.807 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23584.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38603.log 2026-02-20T22:14:21.808 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.33060.log.gz 2026-02-20T22:14:21.808 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51458.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29725.log 2026-02-20T22:14:21.808 INFO:teuthology.orchestra.run.trial107.stderr: 85.2% -- replaced with /var/log/ceph/cluster1-client.mirror.51458.log.gz 2026-02-20T22:14:21.808 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38603.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.38603.log.gz 2026-02-20T22:14:21.808 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49456.log 2026-02-20T22:14:21.814 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29725.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52801.log 2026-02-20T22:14:21.814 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49456.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29725.log.gz 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49456.log.gz 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37491.log 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52801.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38701.log 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.52801.log.gz 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37491.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26169.log 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23584.log.gz 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38701.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37491.log.gz 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47050.log 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.38701.log.gz 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26169.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47802.log 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47050.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49861.log 2026-02-20T22:14:21.815 INFO:teuthology.orchestra.run.trial107.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.26169.log.gz 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47802.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47050.log.gz 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19765.log 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr: 89.5%/var/log/ceph/cluster1-client.mirror.49861.log: -- replaced with /var/log/ceph/cluster2-client.mirror.47802.log.gz 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25677.log 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49861.log.gz 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19765.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26549.log 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr: 64.8% -- replaced with /var/log/ceph/cluster1-client.mirror.19765.log.gz 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25677.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.25677.log.gz 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22702.log 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46235.log 2026-02-20T22:14:21.816 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.22702.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.22702.log.gz 2026-02-20T22:14:21.819 INFO:teuthology.orchestra.run.trial107.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.26549.log.gz 2026-02-20T22:14:21.819 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42718.log 2026-02-20T22:14:21.819 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46235.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39088.log 2026-02-20T22:14:21.819 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42491.log 2026-02-20T22:14:21.819 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39088.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42718.log.gz 2026-02-20T22:14:21.819 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39088.log.gz 2026-02-20T22:14:21.819 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32233.log 2026-02-20T22:14:21.820 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42491.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36480.log 2026-02-20T22:14:21.820 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46235.log.gz 2026-02-20T22:14:21.820 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32233.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42491.log.gz 2026-02-20T22:14:21.834 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47070.log 2026-02-20T22:14:21.834 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36480.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35819.log 2026-02-20T22:14:21.835 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47070.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.36480.log.gz 2026-02-20T22:14:21.835 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31335.log 2026-02-20T22:14:21.839 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35819.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35819.log.gz 2026-02-20T22:14:21.839 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44585.log 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47070.log.gz 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31335.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31335.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54696.log 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44585.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49906.log 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44585.log.gz 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54696.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.54696.log.gz 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36373.log 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49906.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42971.log 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr: 86.9%/var/log/ceph/cluster1-client.mirror.36373.log: -- replaced with /var/log/ceph/cluster1-client.mirror.49906.log.gz 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36373.log.gz 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53480.log 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42971.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24206.log 2026-02-20T22:14:21.840 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42971.log.gz 2026-02-20T22:14:21.841 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30236.log 2026-02-20T22:14:21.842 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24206.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.24206.log.gz 2026-02-20T22:14:21.842 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53352.log 2026-02-20T22:14:21.842 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30236.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53480.log.gz 2026-02-20T22:14:21.842 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52668.log 2026-02-20T22:14:21.842 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53352.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53352.log.gz 2026-02-20T22:14:21.843 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28021.log 2026-02-20T22:14:21.843 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52668.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52668.log.gz 2026-02-20T22:14:21.844 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37981.log 2026-02-20T22:14:21.844 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28021.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28021.log.gz 2026-02-20T22:14:21.844 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26592.log 2026-02-20T22:14:21.845 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37981.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.37981.log.gz 2026-02-20T22:14:21.846 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47299.log 2026-02-20T22:14:21.849 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26592.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26592.log.gz 2026-02-20T22:14:21.850 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36394.log 2026-02-20T22:14:21.850 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.47299.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.47299.log.gz 2026-02-20T22:14:21.851 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52845.log 2026-02-20T22:14:21.852 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36394.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36394.log.gz 2026-02-20T22:14:21.852 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51823.log 2026-02-20T22:14:21.853 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52845.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52845.log.gz 2026-02-20T22:14:21.854 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15890.log 2026-02-20T22:14:21.854 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51823.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.51823.log.gz 2026-02-20T22:14:21.855 INFO:teuthology.orchestra.run.trial107.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.2.32261.log.gz 2026-02-20T22:14:21.855 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36414.log 2026-02-20T22:14:21.855 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.15890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43889.log 2026-02-20T22:14:21.855 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15890.log.gz 2026-02-20T22:14:21.855 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36414.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46215.log 2026-02-20T22:14:21.856 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43889.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43889.log.gz 2026-02-20T22:14:21.857 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50176.log 2026-02-20T22:14:21.857 INFO:teuthology.orchestra.run.trial107.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.36414.log.gz 2026-02-20T22:14:21.857 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33566.log 2026-02-20T22:14:21.857 INFO:teuthology.orchestra.run.trial107.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46215.log.gz 2026-02-20T22:14:21.859 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35954.log 2026-02-20T22:14:21.859 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33566.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50176.log.gz 2026-02-20T22:14:21.859 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.33566.log.gz 2026-02-20T22:14:21.859 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41473.log 2026-02-20T22:14:21.859 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53460.log 2026-02-20T22:14:21.860 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35954.log: /var/log/ceph/cluster2-client.mirror.41473.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.35954.log.gz 2026-02-20T22:14:21.860 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41473.log.gz 2026-02-20T22:14:21.861 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.19464.log 2026-02-20T22:14:21.861 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45067.log 2026-02-20T22:14:21.861 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53460.log: /var/log/ceph/cluster2-client.mirror.1.19464.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53460.log.gz 2026-02-20T22:14:21.861 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49035.log 2026-02-20T22:14:21.861 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45067.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.1.19464.log.gz 2026-02-20T22:14:21.862 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45067.log.gz 2026-02-20T22:14:21.863 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35060.log 2026-02-20T22:14:21.863 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49035.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37749.log 2026-02-20T22:14:21.863 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49035.log.gz 2026-02-20T22:14:21.863 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33492.log 2026-02-20T22:14:21.863 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.35060.log.gz 2026-02-20T22:14:21.863 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37749.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53228.log 2026-02-20T22:14:21.864 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33492.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37749.log.gz 2026-02-20T22:14:21.864 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29546.log 2026-02-20T22:14:21.864 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53228.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.33492.log.gz 2026-02-20T22:14:21.865 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40509.log 2026-02-20T22:14:21.865 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29546.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29546.log.gz 2026-02-20T22:14:21.865 INFO:teuthology.orchestra.run.trial107.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53228.log.gz 2026-02-20T22:14:21.866 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39365.log 2026-02-20T22:14:21.866 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44995.log 2026-02-20T22:14:21.866 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40509.log: /var/log/ceph/cluster1-client.mirror.39365.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39365.log.gz 2026-02-20T22:14:21.868 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40509.log.gz 2026-02-20T22:14:21.868 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46319.log 2026-02-20T22:14:21.868 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52544.log 2026-02-20T22:14:21.868 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44995.log.gz 2026-02-20T22:14:21.869 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46319.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46319.log.gz 2026-02-20T22:14:21.869 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52944.log 2026-02-20T22:14:21.869 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52544.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19833.log 2026-02-20T22:14:21.869 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52944.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52944.log.gz 2026-02-20T22:14:21.869 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38422.log 2026-02-20T22:14:21.870 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19833.log: 56.3% 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52544.log.gz 2026-02-20T22:14:21.870 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.19833.log.gz 2026-02-20T22:14:21.870 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31918.log 2026-02-20T22:14:21.871 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26936.log 2026-02-20T22:14:21.871 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38422.log.gz 2026-02-20T22:14:21.872 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31918.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25005.log 2026-02-20T22:14:21.872 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26936.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31918.log.gz 2026-02-20T22:14:21.872 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26191.log 2026-02-20T22:14:21.873 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.25005.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.25005.log.gz 2026-02-20T22:14:21.873 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41342.log 2026-02-20T22:14:21.874 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26191.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26191.log.gz 2026-02-20T22:14:21.874 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49262.log 2026-02-20T22:14:21.875 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41342.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41342.log.gz 2026-02-20T22:14:21.875 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26936.log.gz 2026-02-20T22:14:21.875 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51233.log 2026-02-20T22:14:21.876 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49262.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47256.log 2026-02-20T22:14:21.876 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51233.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51233.log.gz 2026-02-20T22:14:21.876 INFO:teuthology.orchestra.run.trial107.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49262.log.gz 2026-02-20T22:14:21.876 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30384.log 2026-02-20T22:14:21.877 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47256.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36436.log 2026-02-20T22:14:21.877 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30384.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30384.log.gz 2026-02-20T22:14:21.877 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26330.log 2026-02-20T22:14:21.879 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36436.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47256.log.gz 2026-02-20T22:14:21.879 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36436.log.gz 2026-02-20T22:14:21.879 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29023.log 2026-02-20T22:14:21.880 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26330.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43130.log 2026-02-20T22:14:21.880 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.26330.log.gz 2026-02-20T22:14:21.880 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29023.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30258.log 2026-02-20T22:14:21.881 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43130.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.43130.log.gz 2026-02-20T22:14:21.881 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26443.log 2026-02-20T22:14:21.881 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.30258.log: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.30258.log.gz 2026-02-20T22:14:21.881 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24777.log 2026-02-20T22:14:21.885 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26443.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26443.log.gz 2026-02-20T22:14:21.885 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23103.log 2026-02-20T22:14:21.889 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24777.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24777.log.gz 2026-02-20T22:14:21.889 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50018.log 2026-02-20T22:14:21.890 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23103.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23103.log.gz 2026-02-20T22:14:21.891 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32599.log 2026-02-20T22:14:21.891 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50018.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50018.log.gz 2026-02-20T22:14:21.891 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48029.log 2026-02-20T22:14:21.892 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32599.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32599.log.gz 2026-02-20T22:14:21.892 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48009.log 2026-02-20T22:14:21.894 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48029.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48029.log.gz 2026-02-20T22:14:21.894 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39664.log 2026-02-20T22:14:21.895 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48009.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48009.log.gz 2026-02-20T22:14:21.895 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26787.log 2026-02-20T22:14:21.896 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39664.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39664.log.gz 2026-02-20T22:14:21.897 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35108.log 2026-02-20T22:14:21.898 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26787.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29351.log.gz 2026-02-20T22:14:21.899 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51211.log 2026-02-20T22:14:21.899 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35108.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35108.log.gz 2026-02-20T22:14:21.900 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26063.log 2026-02-20T22:14:21.901 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51211.log: 85.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51211.log.gz 2026-02-20T22:14:21.901 INFO:teuthology.orchestra.run.trial107.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.26787.log.gz 2026-02-20T22:14:21.901 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43320.log 2026-02-20T22:14:21.901 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26063.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45498.log 2026-02-20T22:14:21.901 INFO:teuthology.orchestra.run.trial107.stderr: 88.5%/var/log/ceph/cluster1-client.mirror.43320.log: -- replaced with /var/log/ceph/cluster1-client.mirror.26063.log.gz 2026-02-20T22:14:21.902 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53969.log 2026-02-20T22:14:21.902 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43320.log.gz 2026-02-20T22:14:21.903 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51661.log 2026-02-20T22:14:21.903 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53969.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45498.log.gz 2026-02-20T22:14:21.903 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52223.log 2026-02-20T22:14:21.904 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51661.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53969.log.gz 2026-02-20T22:14:21.904 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51661.log.gz 2026-02-20T22:14:21.904 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33010.log 2026-02-20T22:14:21.905 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52223.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18776.log 2026-02-20T22:14:21.905 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52223.log.gz 2026-02-20T22:14:21.905 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33010.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53329.log 2026-02-20T22:14:21.905 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18776.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.33010.log.gz 2026-02-20T22:14:21.906 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18776.log.gz 2026-02-20T22:14:21.906 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32831.log 2026-02-20T22:14:21.907 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53329.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33614.log 2026-02-20T22:14:21.907 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.32831.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53329.log.gz 2026-02-20T22:14:21.907 INFO:teuthology.orchestra.run.trial107.stderr: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.32831.log.gz 2026-02-20T22:14:21.907 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42126.log 2026-02-20T22:14:21.907 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33614.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42766.log 2026-02-20T22:14:21.907 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33614.log.gz/var/log/ceph/cluster1-client.mirror.42126.log: 2026-02-20T22:14:21.909 INFO:teuthology.orchestra.run.trial107.stderr: 87.8%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.42126.log.gz -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28806.log 2026-02-20T22:14:21.909 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:21.910 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42766.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49613.log 2026-02-20T22:14:21.910 INFO:teuthology.orchestra.run.trial107.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.42766.log.gz 2026-02-20T22:14:21.910 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28806.log: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28806.log.gz 2026-02-20T22:14:21.910 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22382.log 2026-02-20T22:14:21.910 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41065.log 2026-02-20T22:14:21.910 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49613.log.gz 2026-02-20T22:14:21.910 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22382.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.22382.log.gz 2026-02-20T22:14:21.910 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28549.log 2026-02-20T22:14:21.910 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41065.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43200.log 2026-02-20T22:14:21.911 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41065.log.gz 2026-02-20T22:14:21.911 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46802.log 2026-02-20T22:14:21.911 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43200.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43200.log.gz 2026-02-20T22:14:21.911 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47492.log 2026-02-20T22:14:21.912 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46802.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46802.log.gz 2026-02-20T22:14:21.912 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44332.log 2026-02-20T22:14:21.913 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47492.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47492.log.gz 2026-02-20T22:14:21.913 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47029.log 2026-02-20T22:14:21.914 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44332.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52282.log.gz 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44332.log.gz 2026-02-20T22:14:21.914 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:21.914 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48115.log 2026-02-20T22:14:21.914 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22877.log 2026-02-20T22:14:21.915 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48115.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47029.log.gz 2026-02-20T22:14:21.915 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48115.log.gz 2026-02-20T22:14:21.915 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45568.log 2026-02-20T22:14:21.916 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22877.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22877.log.gz 2026-02-20T22:14:21.916 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22542.log 2026-02-20T22:14:21.916 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45568.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23061.log 2026-02-20T22:14:21.916 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22542.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45568.log.gz 2026-02-20T22:14:21.917 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40125.log 2026-02-20T22:14:21.920 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23061.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22542.log.gz 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23061.log.gz 2026-02-20T22:14:21.920 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:21.921 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51301.log 2026-02-20T22:14:21.921 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40125.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44260.log 2026-02-20T22:14:21.921 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51301.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40125.log.gz 2026-02-20T22:14:21.921 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51301.log.gz 2026-02-20T22:14:21.921 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23125.log 2026-02-20T22:14:21.922 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44260.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29292.log 2026-02-20T22:14:21.922 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23125.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44260.log.gz 2026-02-20T22:14:21.922 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22051.log 2026-02-20T22:14:21.923 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29292.log: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29292.log.gz 2026-02-20T22:14:21.923 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48724.log 2026-02-20T22:14:21.927 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22051.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.22051.log.gz 2026-02-20T22:14:21.927 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19680.log 2026-02-20T22:14:21.928 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48724.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48724.log.gz 2026-02-20T22:14:21.928 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50581.log 2026-02-20T22:14:21.928 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19680.log: 71.8% -- replaced with /var/log/ceph/cluster1-client.admin.19680.log.gz 2026-02-20T22:14:21.929 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39136.log 2026-02-20T22:14:21.929 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50581.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50581.log.gz 2026-02-20T22:14:21.929 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54265.log 2026-02-20T22:14:21.930 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39136.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32056.log.gz 2026-02-20T22:14:21.930 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39136.log.gz 2026-02-20T22:14:21.930 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25078.log 2026-02-20T22:14:21.931 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54265.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19312.log 2026-02-20T22:14:21.931 INFO:teuthology.orchestra.run.trial107.stderr: 86.9%/var/log/ceph/cluster2-client.mirror.25078.log: -- replaced with /var/log/ceph/cluster1-client.mirror.54265.log.gz 2026-02-20T22:14:21.931 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49748.log 2026-02-20T22:14:21.932 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19312.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.25078.log.gz 2026-02-20T22:14:21.932 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19312.log.gz 2026-02-20T22:14:21.932 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36800.log 2026-02-20T22:14:21.933 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49748.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49748.log.gz 2026-02-20T22:14:21.933 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36072.log 2026-02-20T22:14:21.933 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36800.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51526.log 2026-02-20T22:14:21.934 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36072.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.36800.log.gz 2026-02-20T22:14:21.934 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36072.log.gz 2026-02-20T22:14:21.934 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24978.log 2026-02-20T22:14:21.935 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51526.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53373.log 2026-02-20T22:14:21.935 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51526.log.gz/var/log/ceph/cluster1-client.admin.24978.log: 2026-02-20T22:14:21.936 INFO:teuthology.orchestra.run.trial107.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.24978.log.gz 2026-02-20T22:14:21.936 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38722.log 2026-02-20T22:14:21.936 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53373.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45043.log 2026-02-20T22:14:21.936 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38722.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53373.log.gz 2026-02-20T22:14:21.937 INFO:teuthology.orchestra.run.trial107.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38722.log.gz 2026-02-20T22:14:21.937 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31357.log 2026-02-20T22:14:21.938 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45043.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38627.log 2026-02-20T22:14:21.938 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31357.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45043.log.gz 2026-02-20T22:14:21.938 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29487.log 2026-02-20T22:14:21.938 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38627.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38627.log.gz 2026-02-20T22:14:21.939 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27732.log 2026-02-20T22:14:21.939 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29487.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.29487.log.gz 2026-02-20T22:14:21.940 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22406.log 2026-02-20T22:14:21.940 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27732.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27732.log.gz 2026-02-20T22:14:21.941 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50378.log 2026-02-20T22:14:21.941 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22406.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.22406.log.gz 2026-02-20T22:14:21.942 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31901.log 2026-02-20T22:14:21.942 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50378.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50378.log.gz 2026-02-20T22:14:21.942 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47947.log 2026-02-20T22:14:21.943 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31901.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31901.log.gz 2026-02-20T22:14:21.943 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35590.log 2026-02-20T22:14:21.944 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47947.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47947.log.gz 2026-02-20T22:14:21.944 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45845.log 2026-02-20T22:14:21.945 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35590.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.35590.log.gz 2026-02-20T22:14:21.945 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52030.log 2026-02-20T22:14:21.946 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45845.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.45845.log.gz 2026-02-20T22:14:21.946 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48455.log 2026-02-20T22:14:21.954 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52030.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52459.log 2026-02-20T22:14:21.954 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48455.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48455.log.gz 2026-02-20T22:14:21.967 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48303.log 2026-02-20T22:14:21.967 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52459.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.52459.log.gz 2026-02-20T22:14:21.987 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.20815.log.gz 2026-02-20T22:14:21.990 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51909.log 2026-02-20T22:14:21.990 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48303.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48303.log.gz 2026-02-20T22:14:21.991 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24318.log 2026-02-20T22:14:21.991 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51909.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51909.log.gz 2026-02-20T22:14:21.991 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19337.log 2026-02-20T22:14:21.994 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24318.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28063.log.gz 2026-02-20T22:14:21.995 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34277.log 2026-02-20T22:14:21.995 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.19337.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.24318.log.gz 2026-02-20T22:14:21.995 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54096.log 2026-02-20T22:14:21.996 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34277.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.34277.log.gz 2026-02-20T22:14:21.996 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.36502.log 2026-02-20T22:14:21.996 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54096.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54096.log.gz 2026-02-20T22:14:21.997 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33036.log 2026-02-20T22:14:22.004 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.1.36502.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.1.36502.log.gz 2026-02-20T22:14:22.004 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43272.log 2026-02-20T22:14:22.005 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33036.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.33036.log.gz 2026-02-20T22:14:22.005 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30587.log 2026-02-20T22:14:22.006 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43272.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43272.log.gz 2026-02-20T22:14:22.006 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19598.log 2026-02-20T22:14:22.007 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30587.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30587.log.gz 2026-02-20T22:14:22.007 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23561.log 2026-02-20T22:14:22.007 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19598.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19598.log.gz 2026-02-20T22:14:22.007 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50221.log 2026-02-20T22:14:22.010 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.23561.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.23561.log.gz 2026-02-20T22:14:22.011 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32014.log 2026-02-20T22:14:22.011 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50221.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50221.log.gz 2026-02-20T22:14:22.012 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54177.log 2026-02-20T22:14:22.012 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32014.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32014.log.gz 2026-02-20T22:14:22.013 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49117.log 2026-02-20T22:14:22.013 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54177.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54177.log.gz 2026-02-20T22:14:22.014 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23626.log 2026-02-20T22:14:22.015 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49117.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49117.log.gz 2026-02-20T22:14:22.016 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52886.log 2026-02-20T22:14:22.017 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23626.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.23626.log.gz 2026-02-20T22:14:22.017 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26486.log 2026-02-20T22:14:22.019 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52886.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52886.log.gz 2026-02-20T22:14:22.019 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46926.log 2026-02-20T22:14:22.023 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26486.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26486.log.gz 2026-02-20T22:14:22.024 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38214.log 2026-02-20T22:14:22.024 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46926.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46926.log.gz 2026-02-20T22:14:22.024 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29975.log 2026-02-20T22:14:22.026 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38214.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38214.log.gz 2026-02-20T22:14:22.027 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24513.log 2026-02-20T22:14:22.027 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29975.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29975.log.gz 2026-02-20T22:14:22.027 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38121.log 2026-02-20T22:14:22.029 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.24513.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.24513.log.gz 2026-02-20T22:14:22.029 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35566.log 2026-02-20T22:14:22.030 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38121.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38121.log.gz 2026-02-20T22:14:22.030 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34120.log 2026-02-20T22:14:22.031 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35566.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.35566.log.gz 2026-02-20T22:14:22.031 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51593.log 2026-02-20T22:14:22.031 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34120.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34120.log.gz 2026-02-20T22:14:22.032 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28922.log 2026-02-20T22:14:22.032 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51593.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51593.log.gz 2026-02-20T22:14:22.032 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47277.log 2026-02-20T22:14:22.033 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28922.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28922.log.gz 2026-02-20T22:14:22.033 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28685.log 2026-02-20T22:14:22.034 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47277.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29747.log.gz 2026-02-20T22:14:22.034 INFO:teuthology.orchestra.run.trial107.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.47277.log.gz 2026-02-20T22:14:22.037 INFO:teuthology.orchestra.run.trial107.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.19337.log.gz 2026-02-20T22:14:22.038 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37029.log 2026-02-20T22:14:22.039 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26916.log 2026-02-20T22:14:22.039 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37029.log: 87.9%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44489.log 2026-02-20T22:14:22.039 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.28685.log.gz 2026-02-20T22:14:22.039 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26916.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49973.log 2026-02-20T22:14:22.039 INFO:teuthology.orchestra.run.trial107.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37029.log.gz 2026-02-20T22:14:22.040 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44489.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44489.log.gz 2026-02-20T22:14:22.041 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46905.log 2026-02-20T22:14:22.041 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49973.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30017.log 2026-02-20T22:14:22.041 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49973.log.gz 2026-02-20T22:14:22.041 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46905.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32191.log 2026-02-20T22:14:22.041 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30017.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46905.log.gz 2026-02-20T22:14:22.041 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30017.log.gz 2026-02-20T22:14:22.041 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29060.log 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33156.log 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29060.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32191.log.gz 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.29060.log.gz 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41547.log 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33156.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53031.log 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41547.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33156.log.gz 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41547.log.gz 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48703.log 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26916.log.gz 2026-02-20T22:14:22.045 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53031.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22901.log 2026-02-20T22:14:22.046 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48703.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49771.log 2026-02-20T22:14:22.046 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22901.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48703.log.gz 2026-02-20T22:14:22.046 INFO:teuthology.orchestra.run.trial107.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53031.log.gz 2026-02-20T22:14:22.046 INFO:teuthology.orchestra.run.trial107.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.22901.log.gz 2026-02-20T22:14:22.046 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51481.log 2026-02-20T22:14:22.051 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49771.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.30258.log 2026-02-20T22:14:22.051 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49771.log.gz 2026-02-20T22:14:22.051 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51481.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18783.log 2026-02-20T22:14:22.051 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51481.log.gz 2026-02-20T22:14:22.051 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36165.log 2026-02-20T22:14:22.051 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.2.30258.log: /var/log/ceph/cluster2-client.admin.18783.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51031.log 2026-02-20T22:14:22.051 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18783.log.gz 2026-02-20T22:14:22.051 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15973.log 2026-02-20T22:14:22.051 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51031.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51031.log.gz 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36165.log.gz 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18932.log 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.15973.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50671.log 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.15973.log.gz 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr: 93.2%/var/log/ceph/cluster2-client.admin.18932.log: -- replaced with /var/log/ceph/cluster2-client.mirror.2.30258.log.gz 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18932.log.gz 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51702.log 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50671.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50918.log 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50671.log.gz 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34831.log 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51702.log: /var/log/ceph/cluster1-client.mirror.50918.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49838.log 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr: 42.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51702.log.gz 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50918.log.gz 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52182.log 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr: 87.5%/var/log/ceph/cluster1-client.mirror.49838.log: -- replaced with /var/log/ceph/cluster1-client.mirror.34831.log.gz 2026-02-20T22:14:22.052 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38374.log 2026-02-20T22:14:22.053 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49838.log.gz 2026-02-20T22:14:22.053 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52182.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45019.log 2026-02-20T22:14:22.053 INFO:teuthology.orchestra.run.trial107.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.52182.log.gz 2026-02-20T22:14:22.054 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38374.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38374.log.gz 2026-02-20T22:14:22.054 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19919.log 2026-02-20T22:14:22.054 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26873.log 2026-02-20T22:14:22.054 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45019.log.gz 2026-02-20T22:14:22.054 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19919.log: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.19919.log.gz 2026-02-20T22:14:22.054 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40652.log 2026-02-20T22:14:22.054 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26873.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22499.log 2026-02-20T22:14:22.054 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40652.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40652.log.gz 2026-02-20T22:14:22.055 INFO:teuthology.orchestra.run.trial107.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21631.log.gz 2026-02-20T22:14:22.055 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50288.log 2026-02-20T22:14:22.055 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22499.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28142.log.gz 2026-02-20T22:14:22.055 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31132.log 2026-02-20T22:14:22.056 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49055.log 2026-02-20T22:14:22.057 INFO:teuthology.orchestra.run.trial107.stderr: 87.0%/var/log/ceph/cluster2-client.mirror.31132.log: -- replaced with /var/log/ceph/cluster1-client.mirror.50288.log.gz 2026-02-20T22:14:22.057 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45091.log 2026-02-20T22:14:22.057 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49055.log: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-20T22:14:22.057 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.26873.log.gz 89.6% 2026-02-20T22:14:22.057 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.31132.log.gz 2026-02-20T22:14:22.057 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45091.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.45091.log.gz 2026-02-20T22:14:22.058 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49055.log.gz 2026-02-20T22:14:22.058 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35769.log 2026-02-20T22:14:22.058 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22499.log.gz 2026-02-20T22:14:22.058 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53738.log 2026-02-20T22:14:22.059 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35769.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35156.log 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39184.log 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53738.log.gz 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.35769.log.gz 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35156.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39776.log 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.35156.log.gz 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30680.log 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39184.log: /var/log/ceph/cluster2-client.mirror.39776.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28744.log 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.39184.log.gz 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30680.log: -- replaced with /var/log/ceph/cluster2-client.mirror.39776.log.gz 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53538.log 2026-02-20T22:14:22.061 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30680.log.gz 2026-02-20T22:14:22.062 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18862.log 2026-02-20T22:14:22.062 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53538.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54385.log 2026-02-20T22:14:22.062 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53538.log.gz 2026-02-20T22:14:22.062 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28744.log.gz 2026-02-20T22:14:22.062 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18862.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18862.log.gzgzip 2026-02-20T22:14:22.063 INFO:teuthology.orchestra.run.trial107.stderr: -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-20T22:14:22.063 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54385.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21522.log 2026-02-20T22:14:22.063 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54385.log.gz 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46657.log 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.21522.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49391.log 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46657.log: 87.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37055.log 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.21522.log.gz 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49391.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46657.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29230.log 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49391.log.gz 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37055.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51436.log 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.29230.log: -- replaced with /var/log/ceph/cluster2-client.mirror.37055.log.gz 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.26191.log 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29102.log.gz 2026-02-20T22:14:22.065 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51436.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37961.log 2026-02-20T22:14:22.066 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29230.log.gz 2026-02-20T22:14:22.066 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.2.26191.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51436.log.gz 2026-02-20T22:14:22.067 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46698.log 2026-02-20T22:14:22.067 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37961.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50603.log 2026-02-20T22:14:22.067 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46698.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49726.log 2026-02-20T22:14:22.067 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50603.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50603.log.gz 2026-02-20T22:14:22.067 INFO:teuthology.orchestra.run.trial107.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.37961.log.gz 2026-02-20T22:14:22.067 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24840.log 2026-02-20T22:14:22.068 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22728.log 2026-02-20T22:14:22.068 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49726.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46698.log.gz 2026-02-20T22:14:22.068 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.49726.log.gz 2026-02-20T22:14:22.068 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24840.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29784.log 2026-02-20T22:14:22.068 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39528.log 2026-02-20T22:14:22.069 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22728.log: /var/log/ceph/cluster2-client.mirror.29784.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.2.26191.log.gz 2026-02-20T22:14:22.069 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41294.log 2026-02-20T22:14:22.069 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29784.log.gz 2026-02-20T22:14:22.069 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39528.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.22728.log.gz 2026-02-20T22:14:22.069 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39528.log.gz 2026-02-20T22:14:22.069 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19262.log 2026-02-20T22:14:22.069 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19411.log 2026-02-20T22:14:22.070 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41294.log: /var/log/ceph/cluster2-client.admin.19262.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45198.log 2026-02-20T22:14:22.070 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19262.log.gz 2026-02-20T22:14:22.070 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.41294.log.gz 2026-02-20T22:14:22.071 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.19411.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33197.log 2026-02-20T22:14:22.071 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51503.log 2026-02-20T22:14:22.071 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45198.log: /var/log/ceph/cluster1-client.mirror.33197.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.24840.log.gz 2026-02-20T22:14:22.071 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45198.log.gz 2026-02-20T22:14:22.071 INFO:teuthology.orchestra.run.trial107.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.23648.log.gz 2026-02-20T22:14:22.072 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27631.log 2026-02-20T22:14:22.072 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51503.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.33197.log.gz 2026-02-20T22:14:22.072 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48327.log 2026-02-20T22:14:22.072 INFO:teuthology.orchestra.run.trial107.stderr: 86.9%/var/log/ceph/cluster2-client.mirror.27631.log: -- replaced with /var/log/ceph/cluster1-client.mirror.51503.log.gz 2026-02-20T22:14:22.073 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33638.log 2026-02-20T22:14:22.073 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48327.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27880.log 2026-02-20T22:14:22.074 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27631.log.gz 2026-02-20T22:14:22.074 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33638.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40901.log 2026-02-20T22:14:22.074 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48327.log.gz 2026-02-20T22:14:22.074 INFO:teuthology.orchestra.run.trial107.stderr:gzip 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.33638.log.gz -5 2026-02-20T22:14:22.074 INFO:teuthology.orchestra.run.trial107.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.48868.log 2026-02-20T22:14:22.074 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27880.log: /var/log/ceph/cluster1-client.mirror.40901.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46256.log 2026-02-20T22:14:22.074 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48868.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21943.log 2026-02-20T22:14:22.074 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.27880.log.gz 2026-02-20T22:14:22.075 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46256.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21787.log 2026-02-20T22:14:22.075 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.40901.log.gz 2026-02-20T22:14:22.075 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46256.log.gz 2026-02-20T22:14:22.075 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.21943.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.21943.log.gz 2026-02-20T22:14:22.075 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48868.log.gz 2026-02-20T22:14:22.075 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25653.log 2026-02-20T22:14:22.076 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.21787.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52645.log 2026-02-20T22:14:22.076 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25653.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.21787.log.gz 2026-02-20T22:14:22.076 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24360.log 2026-02-20T22:14:22.077 INFO:teuthology.orchestra.run.trial107.stderr: 89.4%/var/log/ceph/cluster1-client.mirror.52645.log: -- replaced with /var/log/ceph/cluster1-client.mirror.25653.log.gz 2026-02-20T22:14:22.077 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47905.log 2026-02-20T22:14:22.077 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52645.log.gz 2026-02-20T22:14:22.077 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24360.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25241.log 2026-02-20T22:14:22.077 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47905.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41246.log 2026-02-20T22:14:22.077 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.24360.log.gz 2026-02-20T22:14:22.077 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25241.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26147.log 2026-02-20T22:14:22.078 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26893.log 2026-02-20T22:14:22.078 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41246.log: /var/log/ceph/cluster1-client.mirror.26147.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41246.log.gz 2026-02-20T22:14:22.078 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47905.log.gz 2026-02-20T22:14:22.080 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.25241.log.gz 2026-02-20T22:14:22.081 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26147.log.gz 2026-02-20T22:14:22.081 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15797.log 2026-02-20T22:14:22.082 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26893.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33998.log 2026-02-20T22:14:22.082 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.15797.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24684.log 2026-02-20T22:14:22.082 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15797.log.gz 2026-02-20T22:14:22.082 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32494.log 2026-02-20T22:14:22.083 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51143.log 2026-02-20T22:14:22.083 INFO:teuthology.orchestra.run.trial107.stderr: 90.0%/var/log/ceph/cluster1-client.mirror.24684.log: -- replaced with /var/log/ceph/cluster2-client.mirror.33998.log.gz 2026-02-20T22:14:22.083 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32494.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.24684.log.gz 2026-02-20T22:14:22.085 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26893.log.gz 2026-02-20T22:14:22.085 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25605.log 2026-02-20T22:14:22.086 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51143.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40212.log 2026-02-20T22:14:22.086 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.51143.log.gz 2026-02-20T22:14:22.086 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25605.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53990.log 2026-02-20T22:14:22.086 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.25605.log.gz 2026-02-20T22:14:22.087 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25414.log 2026-02-20T22:14:22.087 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40212.log: /var/log/ceph/cluster1-client.mirror.53990.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.53053.log 2026-02-20T22:14:22.087 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53990.log.gz 2026-02-20T22:14:22.087 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25414.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42790.log 2026-02-20T22:14:22.087 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40212.log.gz 2026-02-20T22:14:22.088 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.1.53053.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19937.log 2026-02-20T22:14:22.088 INFO:teuthology.orchestra.run.trial107.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25414.log.gz 2026-02-20T22:14:22.088 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42790.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.42790.log.gz 2026-02-20T22:14:22.090 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.1.53053.log.gz 2026-02-20T22:14:22.090 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36850.log 2026-02-20T22:14:22.091 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28221.log 2026-02-20T22:14:22.091 INFO:teuthology.orchestra.run.trial107.stderr: 71.8% -- replaced with /var/log/ceph/cluster2-client.admin.19937.log.gz 2026-02-20T22:14:22.091 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36850.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22332.log 2026-02-20T22:14:22.091 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.36850.log.gz 2026-02-20T22:14:22.092 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54646.log 2026-02-20T22:14:22.092 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28221.log: /var/log/ceph/cluster2-client.mirror.22332.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37843.log 2026-02-20T22:14:22.092 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54646.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.54646.log.gz 2026-02-20T22:14:22.092 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.22332.log.gz 2026-02-20T22:14:22.093 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35703.log 2026-02-20T22:14:22.093 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37843.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.39439.log 2026-02-20T22:14:22.093 INFO:teuthology.orchestra.run.trial107.stderr: 87.8%/var/log/ceph/cluster1-client.mirror.35703.log: -- replaced with /var/log/ceph/cluster1-client.mirror.37843.log.gz 2026-02-20T22:14:22.094 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20005.log 2026-02-20T22:14:22.094 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.0.39439.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.35703.log.gz 2026-02-20T22:14:22.095 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41640.log 2026-02-20T22:14:22.095 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.20005.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20005.log.gz 2026-02-20T22:14:22.095 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.0.39439.log.gz 2026-02-20T22:14:22.095 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41113.log 2026-02-20T22:14:22.096 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28648.log 2026-02-20T22:14:22.096 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41113.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41113.log.gz 2026-02-20T22:14:22.096 INFO:teuthology.orchestra.run.trial107.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41640.log.gz 2026-02-20T22:14:22.097 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35219.log 2026-02-20T22:14:22.097 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28648.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52074.log 2026-02-20T22:14:22.097 INFO:teuthology.orchestra.run.trial107.stderr: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28648.log.gz 2026-02-20T22:14:22.098 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35219.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35084.log 2026-02-20T22:14:22.098 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52074.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35219.log.gz 2026-02-20T22:14:22.098 INFO:teuthology.orchestra.run.trial107.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52074.log.gz 2026-02-20T22:14:22.098 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24566.log 2026-02-20T22:14:22.099 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35084.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37609.log 2026-02-20T22:14:22.099 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.35084.log.gz 2026-02-20T22:14:22.099 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24566.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24382.log.gz 2026-02-20T22:14:22.100 INFO:teuthology.orchestra.run.trial107.stderr: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.24566.log.gz 2026-02-20T22:14:22.100 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44718.log 2026-02-20T22:14:22.101 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37609.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39249.log 2026-02-20T22:14:22.101 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37609.log.gz 2026-02-20T22:14:22.101 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23541.log 2026-02-20T22:14:22.101 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39249.log: gzip -5 --verbose -- 87.6% /var/log/ceph/cluster1-client.mirror.39684.log -- replaced with /var/log/ceph/cluster2-client.mirror.44718.log.gz 2026-02-20T22:14:22.101 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.102 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23541.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43680.log 2026-02-20T22:14:22.102 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39684.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.39249.log.gz 2026-02-20T22:14:22.103 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19622.log 2026-02-20T22:14:22.103 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39684.log.gz/var/log/ceph/cluster2-client.mirror.43680.log: 2026-02-20T22:14:22.103 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30646.log 2026-02-20T22:14:22.104 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19622.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.43680.log.gz 2026-02-20T22:14:22.104 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19622.log.gz 2026-02-20T22:14:22.104 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28586.log 2026-02-20T22:14:22.104 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22675.log 2026-02-20T22:14:22.105 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30646.log: /var/log/ceph/cluster2-client.mirror.28586.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23541.log.gz 2026-02-20T22:14:22.106 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28586.log.gz 2026-02-20T22:14:22.117 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35385.log 2026-02-20T22:14:22.118 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54242.log 2026-02-20T22:14:22.118 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.22675.log: /var/log/ceph/cluster1-client.mirror.35385.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.22675.log.gz 2026-02-20T22:14:22.118 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35385.log.gz 2026-02-20T22:14:22.119 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44901.log 2026-02-20T22:14:22.119 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38169.log 2026-02-20T22:14:22.120 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54242.log: /var/log/ceph/cluster2-client.mirror.44901.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54242.log.gz 2026-02-20T22:14:22.120 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20073.log 2026-02-20T22:14:22.120 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.44901.log.gz 2026-02-20T22:14:22.120 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38169.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46029.log 2026-02-20T22:14:22.121 INFO:teuthology.orchestra.run.trial107.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.20073.log: -- replaced with /var/log/ceph/cluster1-client.mirror.38169.log.gz 2026-02-20T22:14:22.121 INFO:teuthology.orchestra.run.trial107.stderr: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.20073.log.gz 2026-02-20T22:14:22.121 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53418.log 2026-02-20T22:14:22.122 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46008.log 2026-02-20T22:14:22.122 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46029.log: /var/log/ceph/cluster1-client.mirror.53418.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46029.log.gz 2026-02-20T22:14:22.122 INFO:teuthology.orchestra.run.trial107.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53418.log.gz 2026-02-20T22:14:22.122 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51800.log 2026-02-20T22:14:22.123 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46008.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30947.log 2026-02-20T22:14:22.123 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51800.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46008.log.gz 2026-02-20T22:14:22.124 INFO:teuthology.orchestra.run.trial107.stderr: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51121.log 2026-02-20T22:14:22.124 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.51800.log.gz 2026-02-20T22:14:22.124 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23038.log 2026-02-20T22:14:22.124 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.30947.log.gz 2026-02-20T22:14:22.125 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51121.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23520.log 2026-02-20T22:14:22.125 INFO:teuthology.orchestra.run.trial107.stderr: 86.9%/var/log/ceph/cluster1-client.mirror.23038.log: -- replaced with /var/log/ceph/cluster1-client.mirror.51121.log.gz 2026-02-20T22:14:22.125 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21567.log 2026-02-20T22:14:22.126 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30488.log.gz/var/log/ceph/cluster1-client.mirror.23520.log: 2026-02-20T22:14:22.126 INFO:teuthology.orchestra.run.trial107.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.23520.log.gz 2026-02-20T22:14:22.126 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38553.log 2026-02-20T22:14:22.126 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21567.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36185.log 2026-02-20T22:14:22.127 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38553.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53308.log.gz 2026-02-20T22:14:22.127 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38553.log.gz 2026-02-20T22:14:22.127 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31086.log 2026-02-20T22:14:22.128 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36185.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39937.log 2026-02-20T22:14:22.128 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31086.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.23038.log.gz 2026-02-20T22:14:22.129 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36325.log 2026-02-20T22:14:22.129 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.36185.log.gz 2026-02-20T22:14:22.129 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39937.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33542.log 2026-02-20T22:14:22.129 INFO:teuthology.orchestra.run.trial107.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.21567.log.gz 2026-02-20T22:14:22.130 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36325.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40101.log 2026-02-20T22:14:22.131 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36325.log.gz 2026-02-20T22:14:22.131 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33542.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.39937.log.gz 2026-02-20T22:14:22.131 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.33542.log.gz 2026-02-20T22:14:22.131 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54307.log 2026-02-20T22:14:22.132 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40101.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50423.log 2026-02-20T22:14:22.132 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54307.log: 87.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27651.log 2026-02-20T22:14:22.132 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.40101.log.gz 2026-02-20T22:14:22.132 INFO:teuthology.orchestra.run.trial107.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54307.log.gz 2026-02-20T22:14:22.132 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50423.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37585.log 2026-02-20T22:14:22.132 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50423.log.gz 2026-02-20T22:14:22.132 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27651.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52582.log 2026-02-20T22:14:22.133 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37585.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28902.log 2026-02-20T22:14:22.133 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37585.log.gz 2026-02-20T22:14:22.133 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.27651.log.gz/var/log/ceph/cluster2-client.mirror.52582.log: 2026-02-20T22:14:22.133 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52582.log.gz 2026-02-20T22:14:22.134 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47409.log 2026-02-20T22:14:22.134 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49703.log 2026-02-20T22:14:22.134 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28902.log: /var/log/ceph/cluster2-client.mirror.47409.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28199.log 2026-02-20T22:14:22.135 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49703.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28902.log.gz 2026-02-20T22:14:22.135 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49703.log.gz 2026-02-20T22:14:22.135 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33745.log 2026-02-20T22:14:22.136 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21450.log 2026-02-20T22:14:22.136 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28199.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47409.log.gz 2026-02-20T22:14:22.136 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33745.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28199.log.gz 2026-02-20T22:14:22.136 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21323.log 2026-02-20T22:14:22.136 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21450.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47513.log 2026-02-20T22:14:22.136 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.33745.log.gz 2026-02-20T22:14:22.136 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.21450.log.gz 2026-02-20T22:14:22.137 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.21323.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42446.log 2026-02-20T22:14:22.137 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47513.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19062.log 2026-02-20T22:14:22.138 INFO:teuthology.orchestra.run.trial107.stderr: 88.4%/var/log/ceph/cluster1-client.mirror.42446.log: -- replaced with /var/log/ceph/cluster2-client.mirror.47513.log.gz 2026-02-20T22:14:22.138 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42446.log.gz 2026-02-20T22:14:22.138 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47091.log 2026-02-20T22:14:22.138 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19062.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19062.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46297.log 2026-02-20T22:14:22.138 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.138 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47091.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32532.log 2026-02-20T22:14:22.139 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46297.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47091.log.gz 2026-02-20T22:14:22.139 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47194.log 2026-02-20T22:14:22.140 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32532.log: 90.5% 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32532.log.gz 2026-02-20T22:14:22.140 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46297.log.gz 2026-02-20T22:14:22.140 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41571.log 2026-02-20T22:14:22.141 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47194.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42877.log 2026-02-20T22:14:22.141 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41571.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41571.log.gz 2026-02-20T22:14:22.142 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49951.log 2026-02-20T22:14:22.142 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42877.log: 95.5% -- replaced with /var/log/ceph/cluster1-client.mirror.21323.log.gz 2026-02-20T22:14:22.143 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26979.log 2026-02-20T22:14:22.143 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47194.log.gz 2026-02-20T22:14:22.143 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42877.log.gz 2026-02-20T22:14:22.143 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49951.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49951.log.gz 2026-02-20T22:14:22.143 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53881.log 2026-02-20T22:14:22.143 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23424.log 2026-02-20T22:14:22.144 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26979.log: /var/log/ceph/cluster2-client.mirror.53881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31840.log 2026-02-20T22:14:22.144 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23424.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53881.log.gz 2026-02-20T22:14:22.144 INFO:teuthology.orchestra.run.trial107.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.23424.log.gz 2026-02-20T22:14:22.145 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41089.log 2026-02-20T22:14:22.145 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31840.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18672.log 2026-02-20T22:14:22.145 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41089.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.31840.log.gz 2026-02-20T22:14:22.146 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41089.log.gz 2026-02-20T22:14:22.146 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38073.log 2026-02-20T22:14:22.146 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18672.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18672.log.gz 2026-02-20T22:14:22.147 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47112.log 2026-02-20T22:14:22.147 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38073.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38073.log.gz 2026-02-20T22:14:22.147 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44859.log 2026-02-20T22:14:22.148 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47112.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.26979.log.gz 2026-02-20T22:14:22.148 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47112.log.gz 2026-02-20T22:14:22.148 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31600.log 2026-02-20T22:14:22.148 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40402.log 2026-02-20T22:14:22.149 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44859.log: /var/log/ceph/cluster2-client.mirror.31600.log: 80.9% -- replaced with /var/log/ceph/cluster2-client.mirror.31600.log.gz 2026-02-20T22:14:22.149 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54889.log 2026-02-20T22:14:22.149 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40402.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27285.log 2026-02-20T22:14:22.150 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.54889.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.40402.log.gz 2026-02-20T22:14:22.150 INFO:teuthology.orchestra.run.trial107.stderr: 92.6% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.54889.log.gz 2026-02-20T22:14:22.150 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.44859.log.gz 2026-02-20T22:14:22.150 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40282.log 2026-02-20T22:14:22.150 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.27285.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18704.log 2026-02-20T22:14:22.151 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40282.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49636.log 2026-02-20T22:14:22.151 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40282.log.gz 2026-02-20T22:14:22.151 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18704.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18704.log.gz 2026-02-20T22:14:22.152 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54721.log 2026-02-20T22:14:22.152 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55214.log 2026-02-20T22:14:22.152 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49636.log.gz 2026-02-20T22:14:22.152 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54721.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.54721.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.27024.log 2026-02-20T22:14:22.152 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.153 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55214.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55214.log.gz 2026-02-20T22:14:22.153 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.19385.log 2026-02-20T22:14:22.153 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.1.27024.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51166.log 2026-02-20T22:14:22.154 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.2.19385.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.27285.log.gz 2026-02-20T22:14:22.154 INFO:teuthology.orchestra.run.trial107.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.27024.log.gz 2026-02-20T22:14:22.154 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28332.log 2026-02-20T22:14:22.155 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23018.log 2026-02-20T22:14:22.155 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51166.log.gz 2026-02-20T22:14:22.155 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25306.log 2026-02-20T22:14:22.155 INFO:teuthology.orchestra.run.trial107.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.2.19385.log.gz 2026-02-20T22:14:22.155 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23018.log: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28332.log.gz 2026-02-20T22:14:22.156 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30817.log 2026-02-20T22:14:22.156 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35998.log 2026-02-20T22:14:22.157 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25306.log: /var/log/ceph/cluster2-client.mirror.30817.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.25306.log.gz 2026-02-20T22:14:22.157 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30817.log.gz 2026-02-20T22:14:22.157 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48744.log 2026-02-20T22:14:22.157 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19212.log 2026-02-20T22:14:22.158 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48744.log: /var/log/ceph/cluster2-client.mirror.35998.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.35998.log.gz 2026-02-20T22:14:22.158 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18854.log 2026-02-20T22:14:22.159 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19212.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19212.log.gz 2026-02-20T22:14:22.159 INFO:teuthology.orchestra.run.trial107.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.23018.log.gz 2026-02-20T22:14:22.159 INFO:teuthology.orchestra.run.trial107.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48744.log.gz 2026-02-20T22:14:22.159 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39750.log 2026-02-20T22:14:22.160 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18854.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46678.log 2026-02-20T22:14:22.160 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.18854.log.gz 2026-02-20T22:14:22.160 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30347.log 2026-02-20T22:14:22.160 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42947.log 2026-02-20T22:14:22.161 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46678.log: /var/log/ceph/cluster2-client.mirror.30347.log: 88.4% 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39750.log.gz 2026-02-20T22:14:22.161 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46678.log.gz 2026-02-20T22:14:22.161 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30347.log.gz 2026-02-20T22:14:22.161 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50716.log 2026-02-20T22:14:22.161 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42947.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51278.log 2026-02-20T22:14:22.162 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42947.log.gz 2026-02-20T22:14:22.162 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50716.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35409.log 2026-02-20T22:14:22.162 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50716.log.gz 2026-02-20T22:14:22.162 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51278.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21474.log 2026-02-20T22:14:22.163 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51278.log.gz 2026-02-20T22:14:22.163 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35409.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35409.log.gz 2026-02-20T22:14:22.163 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28944.log 2026-02-20T22:14:22.163 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-20T22:14:22.164 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.21474.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44838.log 2026-02-20T22:14:22.164 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28944.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.21474.log.gz 2026-02-20T22:14:22.165 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26104.log 2026-02-20T22:14:22.166 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44838.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.44838.log.gz 2026-02-20T22:14:22.167 INFO:teuthology.orchestra.run.trial107.stderr: 87.4% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-20T22:14:22.167 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22031.log 2026-02-20T22:14:22.171 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26104.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26104.log.gz 2026-02-20T22:14:22.171 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25104.log 2026-02-20T22:14:22.171 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22031.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44186.log 2026-02-20T22:14:22.172 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25104.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.25104.log.gz 2026-02-20T22:14:22.173 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40443.log 2026-02-20T22:14:22.174 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44186.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.44186.log.gz 2026-02-20T22:14:22.175 INFO:teuthology.orchestra.run.trial107.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.22031.log.gz 2026-02-20T22:14:22.175 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27940.log 2026-02-20T22:14:22.176 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40443.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.40443.log.gz 2026-02-20T22:14:22.176 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22851.log 2026-02-20T22:14:22.177 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31398.log 2026-02-20T22:14:22.177 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22851.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.27940.log.gz 2026-02-20T22:14:22.178 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43108.log 2026-02-20T22:14:22.178 INFO:teuthology.orchestra.run.trial107.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.22851.log.gz 2026-02-20T22:14:22.178 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.31398.log: 83.3% -- replaced with /var/log/ceph/cluster1-client.mirror.31398.log.gz 2026-02-20T22:14:22.178 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18646.log 2026-02-20T22:14:22.179 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43108.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43477.log 2026-02-20T22:14:22.179 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18646.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18646.log.gz 2026-02-20T22:14:22.180 INFO:teuthology.orchestra.run.trial107.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43108.log.gz 2026-02-20T22:14:22.180 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35843.log 2026-02-20T22:14:22.180 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43477.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25031.log 2026-02-20T22:14:22.181 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43477.log.gz/var/log/ceph/cluster1-client.mirror.35843.log: 2026-02-20T22:14:22.181 INFO:teuthology.orchestra.run.trial107.stderr: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52987.log 2026-02-20T22:14:22.181 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.35843.log.gz 2026-02-20T22:14:22.182 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25031.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25031.log.gz 2026-02-20T22:14:22.182 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40967.log 2026-02-20T22:14:22.182 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52987.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34325.log 2026-02-20T22:14:22.183 INFO:teuthology.orchestra.run.trial107.stderr: 87.5%/var/log/ceph/cluster2-client.mirror.40967.log: -- replaced with /var/log/ceph/cluster1-client.mirror.52987.log.gz 2026-02-20T22:14:22.183 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49303.log 2026-02-20T22:14:22.183 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34325.log: 87.8% 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40967.log.gz 2026-02-20T22:14:22.184 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.34325.log.gz 2026-02-20T22:14:22.184 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31027.log 2026-02-20T22:14:22.184 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49303.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27199.log 2026-02-20T22:14:22.185 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31027.log: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31027.log.gz 2026-02-20T22:14:22.185 INFO:teuthology.orchestra.run.trial107.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49303.log.gz 2026-02-20T22:14:22.186 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32810.log 2026-02-20T22:14:22.186 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.27199.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35912.log 2026-02-20T22:14:22.186 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.32810.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.32810.log.gz 2026-02-20T22:14:22.189 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.27199.log.gz 2026-02-20T22:14:22.189 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26678.log 2026-02-20T22:14:22.190 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27960.log 2026-02-20T22:14:22.191 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35912.log: /var/log/ceph/cluster1-client.mirror.26678.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35912.log.gz 2026-02-20T22:14:22.192 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28470.log 2026-02-20T22:14:22.192 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27960.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.27960.log.gz 2026-02-20T22:14:22.192 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53010.log 2026-02-20T22:14:22.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28470.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26678.log.gz 2026-02-20T22:14:22.194 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47926.log 2026-02-20T22:14:22.194 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53010.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53010.log.gz 2026-02-20T22:14:22.194 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30567.log 2026-02-20T22:14:22.195 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47926.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47926.log.gz 2026-02-20T22:14:22.196 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28448.log 2026-02-20T22:14:22.214 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30567.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43778.log 2026-02-20T22:14:22.214 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28448.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.28448.log.gz 2026-02-20T22:14:22.225 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41407.log 2026-02-20T22:14:22.226 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43778.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43778.log.gz 2026-02-20T22:14:22.234 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51888.log 2026-02-20T22:14:22.235 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41407.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41407.log.gz 2026-02-20T22:14:22.244 INFO:teuthology.orchestra.run.trial107.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.19411.log.gz 2026-02-20T22:14:22.245 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49928.log 2026-02-20T22:14:22.246 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51888.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51888.log.gz 2026-02-20T22:14:22.249 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32452.log 2026-02-20T22:14:22.250 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49928.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49928.log.gz 2026-02-20T22:14:22.251 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52347.log 2026-02-20T22:14:22.251 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32452.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32452.log.gz 2026-02-20T22:14:22.252 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31862.log 2026-02-20T22:14:22.253 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52347.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.52347.log.gz 2026-02-20T22:14:22.253 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54459.log 2026-02-20T22:14:22.257 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31862.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29509.log.gz 2026-02-20T22:14:22.270 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49433.log 2026-02-20T22:14:22.270 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54459.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54459.log.gz 2026-02-20T22:14:22.272 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27156.log 2026-02-20T22:14:22.272 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49433.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49433.log.gz 2026-02-20T22:14:22.272 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34397.log 2026-02-20T22:14:22.274 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.27156.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53560.log.gz 2026-02-20T22:14:22.275 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.27156.log.gz 2026-02-20T22:14:22.276 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27531.log 2026-02-20T22:14:22.276 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34397.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53286.log 2026-02-20T22:14:22.276 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27531.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.34397.log.gz 2026-02-20T22:14:22.278 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33795.log 2026-02-20T22:14:22.278 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27531.log.gz 2026-02-20T22:14:22.278 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53286.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53286.log.gz 2026-02-20T22:14:22.279 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39208.log 2026-02-20T22:14:22.279 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49345.log 2026-02-20T22:14:22.279 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39208.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33795.log.gz 2026-02-20T22:14:22.279 INFO:teuthology.orchestra.run.trial107.stderr: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32552.log 2026-02-20T22:14:22.279 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.39208.log.gz 2026-02-20T22:14:22.283 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32472.log 2026-02-20T22:14:22.283 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49345.log.gz 2026-02-20T22:14:22.283 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32552.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32552.log.gz 2026-02-20T22:14:22.283 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49097.log 2026-02-20T22:14:22.284 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37535.log 2026-02-20T22:14:22.284 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49097.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.32472.log.gz 2026-02-20T22:14:22.284 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49097.log.gz 2026-02-20T22:14:22.284 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.19362.log 2026-02-20T22:14:22.284 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36502.log 2026-02-20T22:14:22.284 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.19362.log: 81.7% -- replaced with /var/log/ceph/cluster2-client.mirror.19362.log.gz 2026-02-20T22:14:22.285 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37535.log.gz 2026-02-20T22:14:22.285 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33219.log 2026-02-20T22:14:22.285 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36502.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40354.log 2026-02-20T22:14:22.285 INFO:teuthology.orchestra.run.trial107.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.36502.log.gz 2026-02-20T22:14:22.285 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33219.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.33219.log.gz 2026-02-20T22:14:22.286 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31659.log.gz 2026-02-20T22:14:22.286 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54746.log 2026-02-20T22:14:22.298 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40354.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33361.log 2026-02-20T22:14:22.298 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54746.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.40354.log.gz 2026-02-20T22:14:22.298 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54746.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21546.log 2026-02-20T22:14:22.298 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.298 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27136.log 2026-02-20T22:14:22.298 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33361.log: /var/log/ceph/cluster1-client.mirror.21546.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20090.log 2026-02-20T22:14:22.299 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27136.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.21546.log.gz 2026-02-20T22:14:22.299 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33361.log.gz 2026-02-20T22:14:22.299 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50446.log 2026-02-20T22:14:22.299 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.20090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46471.log 2026-02-20T22:14:22.299 INFO:teuthology.orchestra.run.trial107.stderr: 56.0% -- replaced with /var/log/ceph/cluster2-client.mirror.20090.log.gz 2026-02-20T22:14:22.299 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50446.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52907.log 2026-02-20T22:14:22.299 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50446.log.gz 2026-02-20T22:14:22.300 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46471.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43248.log 2026-02-20T22:14:22.300 INFO:teuthology.orchestra.run.trial107.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.46471.log.gz 2026-02-20T22:14:22.300 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52907.log: 80.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39824.log 2026-02-20T22:14:22.300 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.52907.log.gz 2026-02-20T22:14:22.300 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43248.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30739.log 2026-02-20T22:14:22.300 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.43248.log.gz 2026-02-20T22:14:22.300 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27136.log.gz 2026-02-20T22:14:22.301 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39824.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.39824.log.gz 2026-02-20T22:14:22.301 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39112.log 2026-02-20T22:14:22.301 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31960.log 2026-02-20T22:14:22.301 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30739.log: /var/log/ceph/cluster1-client.mirror.39112.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45320.log 2026-02-20T22:14:22.301 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39112.log.gz/var/log/ceph/cluster2-client.mirror.31960.log: 2026-02-20T22:14:22.301 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40583.log 2026-02-20T22:14:22.301 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45320.log: 90.2% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-20T22:14:22.301 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45320.log.gz 2026-02-20T22:14:22.308 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36120.log 2026-02-20T22:14:22.308 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24132.log 2026-02-20T22:14:22.309 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40583.log: 87.8%/var/log/ceph/cluster1-client.mirror.36120.log: -- replaced with /var/log/ceph/cluster1-client.mirror.40583.log.gz 2026-02-20T22:14:22.309 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36120.log.gz 2026-02-20T22:14:22.309 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26378.log 2026-02-20T22:14:22.309 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34438.log 2026-02-20T22:14:22.309 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26378.log: 90.1% 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26378.log.gz 2026-02-20T22:14:22.309 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.24132.log.gz 2026-02-20T22:14:22.309 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50648.log 2026-02-20T22:14:22.309 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32686.log 2026-02-20T22:14:22.309 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34438.log: /var/log/ceph/cluster1-client.mirror.50648.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50648.log.gz 2026-02-20T22:14:22.310 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29804.log 2026-02-20T22:14:22.310 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32686.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.34438.log.gz 2026-02-20T22:14:22.310 INFO:teuthology.orchestra.run.trial107.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32686.log.gz 2026-02-20T22:14:22.310 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26400.log 2026-02-20T22:14:22.310 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29804.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37332.log 2026-02-20T22:14:22.310 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29804.log.gz/var/log/ceph/cluster2-client.mirror.26400.log: 2026-02-20T22:14:22.310 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.42533.log 2026-02-20T22:14:22.310 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37332.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.37332.log.gz 2026-02-20T22:14:22.310 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37727.log 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.0.42533.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.26400.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36898.log 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37727.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.37727.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.42533.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44142.log 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36898.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.36898.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41318.log 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47236.log 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44142.log: /var/log/ceph/cluster1-client.mirror.41318.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.27902.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41318.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44142.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51098.log 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46533.log 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51098.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24230.log 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47236.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51098.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46533.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46533.log.gz 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29938.log 2026-02-20T22:14:22.331 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52030.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33819.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24230.log: /var/log/ceph/cluster2-client.mirror.29938.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19902.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29938.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.24230.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33819.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43802.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19902.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42921.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.33819.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.19902.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43802.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38954.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51638.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.43802.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42921.log: /var/log/ceph/cluster1-client.mirror.38954.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24088.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38954.log.gz/var/log/ceph/cluster1-client.mirror.51638.log: 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51844.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51638.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24088.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42921.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42398.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46636.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51844.log: /var/log/ceph/cluster1-client.mirror.42398.log: 91.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23604.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.24088.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42398.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46636.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51844.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22478.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34165.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.23604.log: /var/log/ceph/cluster1-client.mirror.22478.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53518.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46636.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34165.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.22478.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21967.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53518.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44790.log 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.21967.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34165.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.21967.log.gz 2026-02-20T22:14:22.332 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.53518.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51779.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25842.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44790.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51076.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.25842.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44790.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47781.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.25842.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51076.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.23604.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30446.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47781.log: 87.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30134.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.51076.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30446.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39872.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30134.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54155.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.30134.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 89.7%/var/log/ceph/cluster1-client.mirror.39872.log: -- replaced with /var/log/ceph/cluster2-client.mirror.30446.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18549.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.39872.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45439.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47781.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54155.log: /var/log/ceph/cluster2-client.admin.18549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51256.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18549.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45439.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50558.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51256.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38995.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51256.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50558.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46781.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50558.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38995.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48973.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45439.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46781.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28100.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48973.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54796.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28100.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.46781.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48973.log.gz 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46070.log 2026-02-20T22:14:22.333 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.38995.log.gz 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr: 89.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44120.log 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.28100.log.gz 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54796.log: /var/log/ceph/cluster2-client.mirror.46070.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54796.log.gz 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18706.log 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46070.log.gz 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34530.log 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44120.log: /var/log/ceph/cluster2-client.admin.18706.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36621.log 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18706.log.gz 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48889.log 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34530.log: /var/log/ceph/cluster1-client.mirror.36621.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.42212.log 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr: 87.8%/var/log/ceph/cluster2-client.mirror.48889.log: -- replaced with /var/log/ceph/cluster2-client.mirror.34530.log.gz 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.44120.log.gz 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.334 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36621.log.gz 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 89.6% --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.48889.log.gz 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr: -- /var/log/ceph/cluster2-client.mirror.32745.log 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.0.42212.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41682.log 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32745.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19663.log 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41682.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41523.log 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19663.log: 89.9% 80.3% -- replaced with /var/log/ceph/cluster2-client.mirror.41682.log.gz -- replaced with /var/log/ceph/cluster1-client.admin.19663.log.gz 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48579.log 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39389.log 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41523.log: /var/log/ceph/cluster2-client.mirror.48579.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41523.log.gz 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45688.log 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48579.log.gz 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39389.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41595.log 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45688.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39389.log.gz 2026-02-20T22:14:22.335 INFO:teuthology.orchestra.run.trial107.stderr: 87.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27860.log 2026-02-20T22:14:22.336 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.45688.log.gz 2026-02-20T22:14:22.336 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41595.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34048.log 2026-02-20T22:14:22.336 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27860.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41595.log.gz 2026-02-20T22:14:22.336 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25555.log 2026-02-20T22:14:22.336 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27860.log.gz 2026-02-20T22:14:22.337 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34048.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.34048.log.gz 2026-02-20T22:14:22.337 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42470.log 2026-02-20T22:14:22.337 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25555.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.0.42212.log.gz 2026-02-20T22:14:22.337 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49365.log 2026-02-20T22:14:22.338 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41499.log 2026-02-20T22:14:22.338 INFO:teuthology.orchestra.run.trial107.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.25555.log.gz 2026-02-20T22:14:22.338 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49365.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.42470.log.gz 2026-02-20T22:14:22.338 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27923.log 2026-02-20T22:14:22.339 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41499.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53803.log 2026-02-20T22:14:22.339 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41499.log.gz 2026-02-20T22:14:22.339 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.45346.log 2026-02-20T22:14:22.339 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27923.log: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.27923.log.gz 2026-02-20T22:14:22.340 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53803.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46616.log 2026-02-20T22:14:22.340 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.3.45346.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53803.log.gz 2026-02-20T22:14:22.340 INFO:teuthology.orchestra.run.trial107.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49365.log.gz 2026-02-20T22:14:22.341 INFO:teuthology.orchestra.run.trial107.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.3.45346.log.gz 2026-02-20T22:14:22.341 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47153.log 2026-02-20T22:14:22.342 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46616.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53143.log 2026-02-20T22:14:22.342 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47153.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46616.log.gz 2026-02-20T22:14:22.342 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31552.log 2026-02-20T22:14:22.342 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53672.log 2026-02-20T22:14:22.343 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47153.log.gz 2026-02-20T22:14:22.343 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53143.log: /var/log/ceph/cluster2-client.mirror.31552.log: 86.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45821.log 2026-02-20T22:14:22.344 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.53143.log.gz 2026-02-20T22:14:22.344 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53672.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.28243.log 2026-02-20T22:14:22.344 INFO:teuthology.orchestra.run.trial107.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.53672.log.gz 2026-02-20T22:14:22.344 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48136.log 2026-02-20T22:14:22.344 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.1.28243.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.45821.log.gz 2026-02-20T22:14:22.344 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33891.log 2026-02-20T22:14:22.345 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48136.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48136.log.gz 2026-02-20T22:14:22.346 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.28243.log.gz 2026-02-20T22:14:22.346 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-20T22:14:22.346 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33891.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34924.log 2026-02-20T22:14:22.349 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2.audit.log: 87.9% 92.6% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-20T22:14:22.349 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.33891.log.gz 2026-02-20T22:14:22.350 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43383.log 2026-02-20T22:14:22.350 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29329.log 2026-02-20T22:14:22.351 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34924.log: /var/log/ceph/cluster2-client.mirror.43383.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43383.log.gz 2026-02-20T22:14:22.351 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29645.log 2026-02-20T22:14:22.352 INFO:teuthology.orchestra.run.trial107.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34924.log.gz/var/log/ceph/cluster2-client.mirror.29329.log: 2026-02-20T22:14:22.352 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29329.log.gz 2026-02-20T22:14:22.352 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19137.log 2026-02-20T22:14:22.352 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29645.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35542.log 2026-02-20T22:14:22.353 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19137.log: 87.8% 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19137.log.gz 2026-02-20T22:14:22.353 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.29645.log.gz 2026-02-20T22:14:22.353 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32853.log 2026-02-20T22:14:22.354 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35542.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30545.log 2026-02-20T22:14:22.354 INFO:teuthology.orchestra.run.trial107.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35542.log.gz 2026-02-20T22:14:22.355 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32853.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33701.log 2026-02-20T22:14:22.355 INFO:teuthology.orchestra.run.trial107.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32853.log.gz 2026-02-20T22:14:22.355 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30545.log: 87.9%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24338.log 2026-02-20T22:14:22.355 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.30545.log.gz 2026-02-20T22:14:22.355 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33701.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21988.log 2026-02-20T22:14:22.356 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24338.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.33701.log.gz 2026-02-20T22:14:22.356 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48240.log 2026-02-20T22:14:22.359 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21988.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.24338.log.gz 2026-02-20T22:14:22.359 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47967.log 2026-02-20T22:14:22.360 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48240.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48240.log.gz 2026-02-20T22:14:22.360 INFO:teuthology.orchestra.run.trial107.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.21988.log.gz 2026-02-20T22:14:22.361 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21275.log 2026-02-20T22:14:22.362 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47967.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31233.log 2026-02-20T22:14:22.362 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.21275.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.21275.log.gz 2026-02-20T22:14:22.363 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40993.log 2026-02-20T22:14:22.363 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31233.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47967.log.gz 2026-02-20T22:14:22.363 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.31233.log.gz 2026-02-20T22:14:22.364 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26744.log 2026-02-20T22:14:22.364 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40993.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46574.log 2026-02-20T22:14:22.364 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40993.log.gz 2026-02-20T22:14:22.364 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26744.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40077.log 2026-02-20T22:14:22.366 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46574.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46574.log.gz 2026-02-20T22:14:22.367 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39917.log 2026-02-20T22:14:22.367 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40077.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.40077.log.gz 2026-02-20T22:14:22.368 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26744.log.gz 2026-02-20T22:14:22.369 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.19435.log 2026-02-20T22:14:22.370 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39917.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40764.log 2026-02-20T22:14:22.371 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.0.19435.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39917.log.gz 2026-02-20T22:14:22.371 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19748.log 2026-02-20T22:14:22.371 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40764.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40764.log.gz 2026-02-20T22:14:22.372 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19187.log 2026-02-20T22:14:22.372 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19748.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.19748.log.gz 2026-02-20T22:14:22.372 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40170.log 2026-02-20T22:14:22.373 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19187.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19187.log.gz 2026-02-20T22:14:22.373 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21248.log 2026-02-20T22:14:22.374 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40170.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40170.log.gz 2026-02-20T22:14:22.375 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44055.log 2026-02-20T22:14:22.375 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.21248.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.21248.log.gz 2026-02-20T22:14:22.376 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34855.log 2026-02-20T22:14:22.376 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44055.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.0.19435.log.gz 2026-02-20T22:14:22.377 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44055.log.gz 2026-02-20T22:14:22.377 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50626.log 2026-02-20T22:14:22.377 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33954.log 2026-02-20T22:14:22.377 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34855.log: /var/log/ceph/cluster1-client.mirror.50626.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34855.log.gz 2026-02-20T22:14:22.377 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50626.log.gz 2026-02-20T22:14:22.378 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30406.log 2026-02-20T22:14:22.378 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27511.log 2026-02-20T22:14:22.379 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33954.log: /var/log/ceph/cluster2-client.mirror.30406.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.33954.log.gz 2026-02-20T22:14:22.379 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22430.log 2026-02-20T22:14:22.380 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27511.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27511.log.gz 2026-02-20T22:14:22.380 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26084.log 2026-02-20T22:14:22.381 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22430.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.22430.log.gz 2026-02-20T22:14:22.381 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38193.log 2026-02-20T22:14:22.385 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26084.log: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.26084.log.gz 2026-02-20T22:14:22.385 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51053.log 2026-02-20T22:14:22.386 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38193.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38193.log.gz 2026-02-20T22:14:22.386 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42078.log 2026-02-20T22:14:22.387 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51053.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51053.log.gz 2026-02-20T22:14:22.387 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30076.log 2026-02-20T22:14:22.388 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42078.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42078.log.gz 2026-02-20T22:14:22.388 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31803.log 2026-02-20T22:14:22.401 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30076.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.19512.log 2026-02-20T22:14:22.402 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31803.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31803.log.gz 2026-02-20T22:14:22.417 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32574.log 2026-02-20T22:14:22.425 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.3.19512.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29467.log 2026-02-20T22:14:22.433 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32574.log: 93.3%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28885.log 2026-02-20T22:14:22.434 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29467.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29467.log.gz 2026-02-20T22:14:22.441 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53249.log 2026-02-20T22:14:22.442 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28885.log: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28885.log.gz 2026-02-20T22:14:22.449 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.3.19512.log.gz 2026-02-20T22:14:22.449 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46760.log 2026-02-20T22:14:22.450 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53249.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.53249.log.gz 2026-02-20T22:14:22.470 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36597.log 2026-02-20T22:14:22.471 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46760.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46760.log.gz 2026-02-20T22:14:22.489 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29767.log 2026-02-20T22:14:22.490 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36597.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36597.log.gz 2026-02-20T22:14:22.505 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37151.log 2026-02-20T22:14:22.506 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29767.log: 77.4% -- replaced with /var/log/ceph/cluster2-client.mirror.29767.log.gz 2026-02-20T22:14:22.521 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44606.log 2026-02-20T22:14:22.522 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37151.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37151.log.gz 2026-02-20T22:14:22.549 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21813.log 2026-02-20T22:14:22.551 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44606.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.44606.log.gz 2026-02-20T22:14:22.577 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22358.log 2026-02-20T22:14:22.578 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21813.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.21813.log.gz 2026-02-20T22:14:22.586 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32233.log.gz 2026-02-20T22:14:22.594 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37127.log 2026-02-20T22:14:22.594 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22358.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22358.log.gz 2026-02-20T22:14:22.606 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29001.log 2026-02-20T22:14:22.606 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37127.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37127.log.gz 2026-02-20T22:14:22.618 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45869.log 2026-02-20T22:14:22.618 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29001.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29001.log.gz 2026-02-20T22:14:22.633 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34144.log 2026-02-20T22:14:22.635 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45869.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.45869.log.gz 2026-02-20T22:14:22.646 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43730.log 2026-02-20T22:14:22.646 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34144.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34144.log.gz 2026-02-20T22:14:22.657 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41935.log 2026-02-20T22:14:22.658 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43730.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43730.log.gz 2026-02-20T22:14:22.674 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29023.log.gz 2026-02-20T22:14:22.674 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55084.log 2026-02-20T22:14:22.693 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41935.log: 89.7% 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31357.log.gz 2026-02-20T22:14:22.693 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.41935.log.gz 2026-02-20T22:14:22.693 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31637.log 2026-02-20T22:14:22.693 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55084.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41776.log 2026-02-20T22:14:22.694 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.55084.log.gz 2026-02-20T22:14:22.694 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31637.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35725.log 2026-02-20T22:14:22.694 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41776.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31637.log.gz 2026-02-20T22:14:22.694 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.41776.log.gz 2026-02-20T22:14:22.694 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44742.log 2026-02-20T22:14:22.694 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35725.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35725.log.gz 2026-02-20T22:14:22.695 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25436.log 2026-02-20T22:14:22.695 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44742.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45112.log 2026-02-20T22:14:22.695 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.44742.log.gz 2026-02-20T22:14:22.695 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25436.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.25436.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48558.log 2026-02-20T22:14:22.695 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.695 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45112.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.45112.log.gz 2026-02-20T22:14:22.696 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41270.log 2026-02-20T22:14:22.696 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48558.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21609.log 2026-02-20T22:14:22.696 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41270.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.41270.log.gz 2026-02-20T22:14:22.696 INFO:teuthology.orchestra.run.trial107.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48558.log.gz 2026-02-20T22:14:22.696 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46111.log 2026-02-20T22:14:22.696 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.21609.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33771.log 2026-02-20T22:14:22.697 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46111.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.21609.log.gz 2026-02-20T22:14:22.697 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30624.log 2026-02-20T22:14:22.697 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.33771.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.33771.log.gz 2026-02-20T22:14:22.697 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36207.log 2026-02-20T22:14:22.697 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30624.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46111.log.gz 2026-02-20T22:14:22.697 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.30624.log.gz 2026-02-20T22:14:22.697 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28981.log 2026-02-20T22:14:22.698 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36207.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34626.log 2026-02-20T22:14:22.698 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28981.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.36207.log.gz 2026-02-20T22:14:22.698 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54531.log 2026-02-20T22:14:22.698 INFO:teuthology.orchestra.run.trial107.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28981.log.gz/var/log/ceph/cluster1-client.mirror.34626.log: 2026-02-20T22:14:22.698 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.34626.log.gz 2026-02-20T22:14:22.698 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19850.log 2026-02-20T22:14:22.699 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54531.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46430.log 2026-02-20T22:14:22.699 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54531.log.gz 2026-02-20T22:14:22.699 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19850.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19850.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38858.log 2026-02-20T22:14:22.699 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.699 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46430.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46430.log.gz 2026-02-20T22:14:22.699 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.46319.log 2026-02-20T22:14:22.700 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38858.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54407.log 2026-02-20T22:14:22.700 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.38858.log.gz 2026-02-20T22:14:22.700 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.2.46319.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26127.log 2026-02-20T22:14:22.701 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54407.log: 94.1% -- replaced with /var/log/ceph/cluster1-client.mirror.2.46319.log.gz 2026-02-20T22:14:22.706 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38047.log 2026-02-20T22:14:22.710 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26127.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.26127.log.gz 2026-02-20T22:14:22.710 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24540.log 2026-02-20T22:14:22.711 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38047.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.38047.log.gz 2026-02-20T22:14:22.711 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19697.log 2026-02-20T22:14:22.712 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.24540.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.24540.log.gz 2026-02-20T22:14:22.712 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27242.log 2026-02-20T22:14:22.713 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19697.log: 63.2% -- replaced with /var/log/ceph/cluster1-client.mirror.19697.log.gz 2026-02-20T22:14:22.713 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29450.log 2026-02-20T22:14:22.717 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.27242.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.27242.log.gz 2026-02-20T22:14:22.717 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53761.log 2026-02-20T22:14:22.717 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29450.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29450.log.gz 2026-02-20T22:14:22.718 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28120.log 2026-02-20T22:14:22.718 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53761.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.53761.log.gz 2026-02-20T22:14:22.718 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46554.log 2026-02-20T22:14:22.719 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28120.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28120.log.gz 2026-02-20T22:14:22.719 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41752.log 2026-02-20T22:14:22.720 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46554.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46554.log.gz 2026-02-20T22:14:22.720 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32367.log 2026-02-20T22:14:22.721 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41752.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41752.log.gz 2026-02-20T22:14:22.721 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47533.log 2026-02-20T22:14:22.722 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32367.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32367.log.gz 2026-02-20T22:14:22.723 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27671.log 2026-02-20T22:14:22.724 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47533.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47533.log.gz 2026-02-20T22:14:22.724 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42374.log 2026-02-20T22:14:22.726 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27671.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27671.log.gz 2026-02-20T22:14:22.726 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35932.log 2026-02-20T22:14:22.726 INFO:teuthology.orchestra.run.trial107.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30236.log.gz 2026-02-20T22:14:22.727 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42374.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47885.log 2026-02-20T22:14:22.727 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35932.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.42374.log.gz 2026-02-20T22:14:22.727 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38579.log 2026-02-20T22:14:22.728 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47885.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.35932.log.gz 2026-02-20T22:14:22.728 INFO:teuthology.orchestra.run.trial107.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47885.log.gz 2026-02-20T22:14:22.728 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43847.log 2026-02-20T22:14:22.729 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37867.log 2026-02-20T22:14:22.729 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38579.log: /var/log/ceph/cluster2-client.mirror.43847.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38579.log.gz 2026-02-20T22:14:22.729 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44945.log 2026-02-20T22:14:22.730 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37867.log: 92.7% 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37867.log.gz 2026-02-20T22:14:22.730 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.43847.log.gz 2026-02-20T22:14:22.730 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49546.log 2026-02-20T22:14:22.731 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27265.log 2026-02-20T22:14:22.731 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44945.log: /var/log/ceph/cluster1-client.mirror.49546.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49546.log.gz 2026-02-20T22:14:22.731 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.44945.log.gz 2026-02-20T22:14:22.731 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31255.log 2026-02-20T22:14:22.731 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27265.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.19411.log 2026-02-20T22:14:22.734 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31255.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27265.log.gz 2026-02-20T22:14:22.736 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31862.log.gz 2026-02-20T22:14:22.743 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30567.log.gz 2026-02-20T22:14:22.745 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37686.log 2026-02-20T22:14:22.746 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35177.log 2026-02-20T22:14:22.746 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.3.19411.log: /var/log/ceph/cluster1-client.mirror.37686.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27818.log 2026-02-20T22:14:22.746 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.37686.log.gz 2026-02-20T22:14:22.747 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53716.log 2026-02-20T22:14:22.747 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.27818.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.27818.log.gz 2026-02-20T22:14:22.748 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35177.log.gz 2026-02-20T22:14:22.754 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.19467.log 2026-02-20T22:14:22.754 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53716.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54055.log 2026-02-20T22:14:22.755 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.2.19467.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53716.log.gz 2026-02-20T22:14:22.755 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52482.log 2026-02-20T22:14:22.756 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54055.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54055.log.gz 2026-02-20T22:14:22.756 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20056.log 2026-02-20T22:14:22.756 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52482.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.52482.log.gz 2026-02-20T22:14:22.757 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36144.log 2026-02-20T22:14:22.757 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.20056.log: 65.1% -- replaced with /var/log/ceph/cluster2-client.mirror.20056.log.gz 2026-02-20T22:14:22.757 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47215.log 2026-02-20T22:14:22.758 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36144.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.36144.log.gz 2026-02-20T22:14:22.758 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27471.log 2026-02-20T22:14:22.759 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47215.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47215.log.gz 2026-02-20T22:14:22.759 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53187.log 2026-02-20T22:14:22.760 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27471.log: 94.2% 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27471.log.gz 2026-02-20T22:14:22.760 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.2.19467.log.gz 2026-02-20T22:14:22.760 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48910.log 2026-02-20T22:14:22.761 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53187.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47008.log 2026-02-20T22:14:22.761 INFO:teuthology.orchestra.run.trial107.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53187.log.gz 2026-02-20T22:14:22.761 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48910.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52095.log 2026-02-20T22:14:22.762 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47008.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48910.log.gz 2026-02-20T22:14:22.762 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47430.log 2026-02-20T22:14:22.763 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52095.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47008.log.gz 2026-02-20T22:14:22.763 INFO:teuthology.orchestra.run.trial107.stderr: 91.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36922.log 2026-02-20T22:14:22.763 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.52095.log.gz 2026-02-20T22:14:22.764 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49200.log 2026-02-20T22:14:22.764 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47430.log.gz 2026-02-20T22:14:22.764 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32616.log 2026-02-20T22:14:22.765 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49200.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36922.log.gz 2026-02-20T22:14:22.765 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42511.log 2026-02-20T22:14:22.765 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49200.log.gz 2026-02-20T22:14:22.765 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51950.log 2026-02-20T22:14:22.766 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42511.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.32616.log.gz 2026-02-20T22:14:22.767 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52966.log 2026-02-20T22:14:22.767 INFO:teuthology.orchestra.run.trial107.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.42511.log.gz 2026-02-20T22:14:22.767 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51950.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40029.log 2026-02-20T22:14:22.769 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52966.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51950.log.gz 2026-02-20T22:14:22.769 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18594.log 2026-02-20T22:14:22.770 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40029.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.40029.log.gz 2026-02-20T22:14:22.770 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48662.log 2026-02-20T22:14:22.770 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18594.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18594.log.gz 2026-02-20T22:14:22.771 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39554.log 2026-02-20T22:14:22.771 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48662.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48662.log.gz 2026-02-20T22:14:22.772 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47678.log 2026-02-20T22:14:22.772 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39554.log: 78.3% -- replaced with /var/log/ceph/cluster1-client.mirror.39554.log.gz 2026-02-20T22:14:22.773 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35430.log 2026-02-20T22:14:22.773 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47678.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28944.log.gz 2026-02-20T22:14:22.773 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37103.log 2026-02-20T22:14:22.774 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47678.log.gz 2026-02-20T22:14:22.774 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35430.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35132.log 2026-02-20T22:14:22.775 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37103.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37103.log.gz 2026-02-20T22:14:22.775 INFO:teuthology.orchestra.run.trial107.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35430.log.gz 2026-02-20T22:14:22.775 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32494.log.gz 2026-02-20T22:14:22.775 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27024.log 2026-02-20T22:14:22.775 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50896.log 2026-02-20T22:14:22.776 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54133.log 2026-02-20T22:14:22.776 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster2-client.mirror.27024.log.gz 2026-02-20T22:14:22.776 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.35132.log.gz 2026-02-20T22:14:22.776 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50896.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47595.log 2026-02-20T22:14:22.776 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50896.log.gz 2026-02-20T22:14:22.776 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25629.log 2026-02-20T22:14:22.777 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.54133.log.gz 2026-02-20T22:14:22.777 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34554.log 2026-02-20T22:14:22.777 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47595.log: /var/log/ceph/cluster1-client.mirror.25629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31938.log 2026-02-20T22:14:22.777 INFO:teuthology.orchestra.run.trial107.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.25629.log.gz 2026-02-20T22:14:22.778 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34554.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42533.log 2026-02-20T22:14:22.778 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34554.log.gz 2026-02-20T22:14:22.778 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31938.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31938.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27795.log 2026-02-20T22:14:22.778 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.779 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47595.log.gz 2026-02-20T22:14:22.779 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42533.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39848.log 2026-02-20T22:14:22.779 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.27795.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.42533.log.gz 2026-02-20T22:14:22.779 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38326.log 2026-02-20T22:14:22.779 INFO:teuthology.orchestra.run.trial107.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.27795.log.gz 2026-02-20T22:14:22.780 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31768.log 2026-02-20T22:14:22.780 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.39848.log: /var/log/ceph/cluster2-client.mirror.38326.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50063.log 2026-02-20T22:14:22.780 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.39848.log.gz 2026-02-20T22:14:22.780 INFO:teuthology.orchestra.run.trial107.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38326.log.gz 2026-02-20T22:14:22.780 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31768.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50941.log 2026-02-20T22:14:22.781 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50063.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50063.log.gz 2026-02-20T22:14:22.781 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46049.log 2026-02-20T22:14:22.782 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26658.log 2026-02-20T22:14:22.782 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50941.log: /var/log/ceph/cluster2-client.mirror.46049.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50941.log.gz 2026-02-20T22:14:22.783 INFO:teuthology.orchestra.run.trial107.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46049.log.gz 2026-02-20T22:14:22.789 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23297.log 2026-02-20T22:14:22.790 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26658.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.15860.log 2026-02-20T22:14:22.790 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.23297.log: 78.1% -- replaced with /var/log/ceph/cluster1-client.mirror.23297.log.gz 2026-02-20T22:14:22.791 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42054.log 2026-02-20T22:14:22.791 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.15860.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.15860.log.gz 2026-02-20T22:14:22.791 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24862.log 2026-02-20T22:14:22.792 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42054.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.42054.log.gz 2026-02-20T22:14:22.792 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22750.log 2026-02-20T22:14:22.793 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26658.log.gz 2026-02-20T22:14:22.793 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24862.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53208.log 2026-02-20T22:14:22.794 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22750.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.22750.log.gz 2026-02-20T22:14:22.794 INFO:teuthology.orchestra.run.trial107.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.24862.log.gz 2026-02-20T22:14:22.794 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47761.log 2026-02-20T22:14:22.794 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53208.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30525.log 2026-02-20T22:14:22.794 INFO:teuthology.orchestra.run.trial107.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53208.log.gz 2026-02-20T22:14:22.795 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19731.log 2026-02-20T22:14:22.795 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47761.log.gz 2026-02-20T22:14:22.795 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30525.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29588.log 2026-02-20T22:14:22.796 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30525.log.gz 2026-02-20T22:14:22.796 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19731.log: 4.7% -- replaced with /var/log/ceph/cluster1-client.mirror.19731.log.gz 2026-02-20T22:14:22.796 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51346.log 2026-02-20T22:14:22.796 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29588.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25741.log 2026-02-20T22:14:22.797 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51346.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.51346.log.gz 2026-02-20T22:14:22.797 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30800.log 2026-02-20T22:14:22.801 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25741.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25741.log.gz 2026-02-20T22:14:22.801 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40003.log 2026-02-20T22:14:22.802 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30800.log: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30800.log.gz 2026-02-20T22:14:22.802 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49241.log 2026-02-20T22:14:22.803 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40003.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40003.log.gz 2026-02-20T22:14:22.803 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23222.log 2026-02-20T22:14:22.805 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49241.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30406.log.gz 2026-02-20T22:14:22.805 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43224.log 2026-02-20T22:14:22.806 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.23222.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49241.log.gz 2026-02-20T22:14:22.806 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40465.log 2026-02-20T22:14:22.806 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.23222.log.gz 2026-02-20T22:14:22.806 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43224.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.43224.log.gz 2026-02-20T22:14:22.807 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51188.log 2026-02-20T22:14:22.807 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21760.log 2026-02-20T22:14:22.808 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40465.log: /var/log/ceph/cluster1-client.mirror.51188.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51188.log.gz 2026-02-20T22:14:22.808 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40465.log.gz 2026-02-20T22:14:22.808 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45346.log 2026-02-20T22:14:22.808 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30739.log.gz 2026-02-20T22:14:22.809 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29566.log 2026-02-20T22:14:22.809 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.21760.log: /var/log/ceph/cluster1-client.mirror.45346.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.45346.log.gz 2026-02-20T22:14:22.809 INFO:teuthology.orchestra.run.trial107.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.21760.log.gz 2026-02-20T22:14:22.809 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44561.log 2026-02-20T22:14:22.810 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18940.log 2026-02-20T22:14:22.810 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29566.log: /var/log/ceph/cluster1-client.mirror.44561.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48517.log 2026-02-20T22:14:22.810 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18940.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29566.log.gz 2026-02-20T22:14:22.810 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44561.log.gz 2026-02-20T22:14:22.811 INFO:teuthology.orchestra.run.trial107.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18940.log.gz 2026-02-20T22:14:22.811 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25991.log 2026-02-20T22:14:22.811 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50536.log 2026-02-20T22:14:22.811 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48517.log: /var/log/ceph/cluster1-client.mirror.25991.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53395.log 2026-02-20T22:14:22.811 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50536.log: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.25991.log.gz 2026-02-20T22:14:22.812 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50536.log.gz 2026-02-20T22:14:22.812 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48517.log.gz 2026-02-20T22:14:22.812 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47988.log 2026-02-20T22:14:22.812 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51616.log 2026-02-20T22:14:22.812 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53395.log: /var/log/ceph/cluster2-client.mirror.47988.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27711.log 2026-02-20T22:14:22.812 INFO:teuthology.orchestra.run.trial107.stderr: 87.0%/var/log/ceph/cluster1-client.mirror.51616.log: -- replaced with /var/log/ceph/cluster1-client.mirror.53395.log.gz 2026-02-20T22:14:22.813 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51616.log.gz 2026-02-20T22:14:22.813 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47988.log.gz 2026-02-20T22:14:22.813 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42167.log 2026-02-20T22:14:22.813 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27711.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27711.log.gz 2026-02-20T22:14:22.814 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50356.log 2026-02-20T22:14:22.814 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49478.log 2026-02-20T22:14:22.815 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42167.log: /var/log/ceph/cluster1-client.mirror.50356.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32515.log 2026-02-20T22:14:22.815 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50356.log.gz/var/log/ceph/cluster1-client.mirror.49478.log: 2026-02-20T22:14:22.815 INFO:teuthology.orchestra.run.trial107.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.49478.log.gz 2026-02-20T22:14:22.815 INFO:teuthology.orchestra.run.trial107.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.42167.log.gz 2026-02-20T22:14:22.816 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54436.log 2026-02-20T22:14:22.816 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39959.log 2026-02-20T22:14:22.816 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32515.log: /var/log/ceph/cluster1-client.mirror.54436.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52711.log 2026-02-20T22:14:22.816 INFO:teuthology.orchestra.run.trial107.stderr: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.32515.log.gz 2026-02-20T22:14:22.817 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39959.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54436.log.gz 2026-02-20T22:14:22.817 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52260.log 2026-02-20T22:14:22.817 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52711.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39959.log.gz 2026-02-20T22:14:22.817 INFO:teuthology.orchestra.run.trial107.stderr: 84.1% -- replaced with /var/log/ceph/cluster2-client.mirror.52711.log.gz 2026-02-20T22:14:22.817 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38742.log 2026-02-20T22:14:22.818 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30508.log 2026-02-20T22:14:22.818 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52260.log: /var/log/ceph/cluster1-client.mirror.38742.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51413.log 2026-02-20T22:14:22.818 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30508.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52260.log.gz 2026-02-20T22:14:22.819 INFO:teuthology.orchestra.run.trial107.stderr: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30508.log.gz 2026-02-20T22:14:22.819 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20129.log 2026-02-20T22:14:22.819 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27451.log 2026-02-20T22:14:22.819 INFO:teuthology.orchestra.run.trial107.stderr: 92.5%/var/log/ceph/cluster1-client.mirror.51413.log: -- replaced with /var/log/ceph/cluster1-client.mirror.38742.log.gz 2026-02-20T22:14:22.819 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.20129.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51413.log.gz 2026-02-20T22:14:22.820 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49014.log 2026-02-20T22:14:22.820 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.20129.log.gz 2026-02-20T22:14:22.820 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45941.log 2026-02-20T22:14:22.820 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27451.log: /var/log/ceph/cluster2-client.mirror.49014.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27451.log.gz 2026-02-20T22:14:22.820 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49014.log.gz 2026-02-20T22:14:22.821 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40190.log 2026-02-20T22:14:22.821 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30054.log 2026-02-20T22:14:22.821 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45941.log: /var/log/ceph/cluster1-client.mirror.40190.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52524.log 2026-02-20T22:14:22.821 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30054.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45941.log.gz 2026-02-20T22:14:22.822 INFO:teuthology.orchestra.run.trial107.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.30054.log.gz 2026-02-20T22:14:22.822 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51368.log 2026-02-20T22:14:22.822 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52524.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.40190.log.gz 2026-02-20T22:14:22.822 INFO:teuthology.orchestra.run.trial107.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52524.log.gz 2026-02-20T22:14:22.823 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36024.log 2026-02-20T22:14:22.823 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26354.log 2026-02-20T22:14:22.823 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51368.log: /var/log/ceph/cluster2-client.mirror.36024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54365.log 2026-02-20T22:14:22.824 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.51368.log.gz/var/log/ceph/cluster1-client.mirror.26354.log: 2026-02-20T22:14:22.824 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36024.log.gz 2026-02-20T22:14:22.824 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49324.log 2026-02-20T22:14:22.824 INFO:teuthology.orchestra.run.trial107.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26354.log.gz 2026-02-20T22:14:22.824 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39341.log 2026-02-20T22:14:22.824 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.54365.log: /var/log/ceph/cluster2-client.mirror.49324.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51971.log 2026-02-20T22:14:22.825 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39341.log: 87.9% 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49324.log.gz 2026-02-20T22:14:22.825 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.39341.log.gz 2026-02-20T22:14:22.825 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54365.log.gz 2026-02-20T22:14:22.825 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37469.log 2026-02-20T22:14:22.826 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31064.log 2026-02-20T22:14:22.826 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51971.log: /var/log/ceph/cluster1-client.mirror.37469.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29213.log 2026-02-20T22:14:22.826 INFO:teuthology.orchestra.run.trial107.stderr: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51971.log.gz 2026-02-20T22:14:22.826 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31064.log: 87.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18705.log 2026-02-20T22:14:22.827 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.31064.log.gz 2026-02-20T22:14:22.827 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29213.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.29213.log.gz 2026-02-20T22:14:22.827 INFO:teuthology.orchestra.run.trial107.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.37469.log.gz 2026-02-20T22:14:22.827 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34185.log 2026-02-20T22:14:22.827 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36943.log 2026-02-20T22:14:22.828 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18705.log: /var/log/ceph/cluster1-client.mirror.34185.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18705.log.gz 2026-02-20T22:14:22.828 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30364.log 2026-02-20T22:14:22.828 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.19337.log 2026-02-20T22:14:22.829 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36943.log: /var/log/ceph/cluster2-client.mirror.30364.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.34185.log.gz 2026-02-20T22:14:22.829 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.30364.log.gz 2026-02-20T22:14:22.829 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36943.log.gz 2026-02-20T22:14:22.830 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37282.log 2026-02-20T22:14:22.830 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37940.log 2026-02-20T22:14:22.830 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.0.19337.log: /var/log/ceph/cluster2-client.mirror.37282.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41913.log 2026-02-20T22:14:22.831 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37940.log: 87.7% 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37282.log.gz 2026-02-20T22:14:22.831 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.37940.log.gz 2026-02-20T22:14:22.841 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.32853.log 2026-02-20T22:14:22.842 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19287.log 2026-02-20T22:14:22.843 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41913.log: /var/log/ceph/cluster1-client.mirror.2.32853.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.41913.log.gz 2026-02-20T22:14:22.843 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47740.log 2026-02-20T22:14:22.844 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19287.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19287.log.gz 2026-02-20T22:14:22.844 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39315.log 2026-02-20T22:14:22.845 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47740.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47740.log.gz 2026-02-20T22:14:22.845 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34301.log 2026-02-20T22:14:22.846 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39315.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.39315.log.gz 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.2.32853.log.gz 2026-02-20T22:14:22.846 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:22.847 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35010.log 2026-02-20T22:14:22.847 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34301.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46595.log 2026-02-20T22:14:22.847 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.34301.log.gz 2026-02-20T22:14:22.848 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35010.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55032.log 2026-02-20T22:14:22.848 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46595.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.35010.log.gz 2026-02-20T22:14:22.848 INFO:teuthology.orchestra.run.trial107.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46595.log.gz 2026-02-20T22:14:22.848 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52202.log 2026-02-20T22:14:22.849 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.55032.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.55032.log.gz 2026-02-20T22:14:22.849 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28004.log 2026-02-20T22:14:22.849 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51988.log 2026-02-20T22:14:22.850 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52202.log: /var/log/ceph/cluster2-client.mirror.28004.log: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28004.log.gz 2026-02-20T22:14:22.850 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26304.log 2026-02-20T22:14:22.851 INFO:teuthology.orchestra.run.trial107.stderr: 94.3%/var/log/ceph/cluster2-client.mirror.51988.log: -- replaced with /var/log/ceph/cluster2-client.mirror.28470.log.gz 2026-02-20T22:14:22.851 INFO:teuthology.orchestra.run.trial107.stderr: 89.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27691.log 2026-02-20T22:14:22.851 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.51988.log.gz 2026-02-20T22:14:22.852 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26304.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52202.log.gz 2026-02-20T22:14:22.852 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18861.log 2026-02-20T22:14:22.852 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27691.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53823.log 2026-02-20T22:14:22.852 INFO:teuthology.orchestra.run.trial107.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.26304.log.gz 2026-02-20T22:14:22.852 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18861.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.27691.log.gz 2026-02-20T22:14:22.853 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18861.log.gz 2026-02-20T22:14:22.853 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.23125.log.gz 2026-02-20T22:14:22.853 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22247.log 2026-02-20T22:14:22.853 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53823.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42212.log 2026-02-20T22:14:22.854 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.22247.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29826.log 2026-02-20T22:14:22.854 INFO:teuthology.orchestra.run.trial107.stderr: 87.7%/var/log/ceph/cluster1-client.mirror.42212.log: -- replaced with /var/log/ceph/cluster1-client.admin.22247.log.gz 2026-02-20T22:14:22.854 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46492.log 2026-02-20T22:14:22.854 INFO:teuthology.orchestra.run.trial107.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42212.log.gz 2026-02-20T22:14:22.854 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29826.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53823.log.gz 2026-02-20T22:14:22.854 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30859.log 2026-02-20T22:14:22.855 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46492.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46843.log 2026-02-20T22:14:22.855 INFO:teuthology.orchestra.run.trial107.stderr: 88.4%/var/log/ceph/cluster2-client.mirror.30859.log: -- replaced with /var/log/ceph/cluster2-client.mirror.46492.log.gz 2026-02-20T22:14:22.869 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26463.log 2026-02-20T22:14:22.870 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46843.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47843.log 2026-02-20T22:14:22.870 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26463.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46843.log.gz 2026-02-20T22:14:22.871 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38145.log 2026-02-20T22:14:22.872 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47843.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.47843.log.gz 2026-02-20T22:14:22.873 INFO:teuthology.orchestra.run.trial107.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.19337.log.gz 2026-02-20T22:14:22.873 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47554.log 2026-02-20T22:14:22.873 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29997.log 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47554.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.38145.log.gz 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26463.log.gz 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50851.log 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47554.log.gz 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29997.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40923.log 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50851.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50153.log 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50851.log.gz/var/log/ceph/cluster2-client.mirror.40923.log: 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54508.log 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50153.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.40923.log.gz 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50153.log.gz 2026-02-20T22:14:22.876 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40535.log 2026-02-20T22:14:22.877 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54508.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53501.log 2026-02-20T22:14:22.877 INFO:teuthology.orchestra.run.trial107.stderr: 86.9%/var/log/ceph/cluster2-client.mirror.40535.log: -- replaced with /var/log/ceph/cluster1-client.mirror.54508.log.gz 2026-02-20T22:14:22.877 INFO:teuthology.orchestra.run.trial107.stderr: 87.9%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53903.log 2026-02-20T22:14:22.878 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.40535.log.gz 2026-02-20T22:14:22.878 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53501.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32034.log 2026-02-20T22:14:22.878 INFO:teuthology.orchestra.run.trial107.stderr: 80.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53501.log.gz 2026-02-20T22:14:22.878 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.53903.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46277.log 2026-02-20T22:14:22.878 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32034.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.32034.log.gz 2026-02-20T22:14:22.878 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34783.log 2026-02-20T22:14:22.879 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46277.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46277.log.gz 2026-02-20T22:14:22.879 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49501.log 2026-02-20T22:14:22.879 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34783.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.34783.log.gz 2026-02-20T22:14:22.880 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34460.log 2026-02-20T22:14:22.880 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49501.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49501.log.gz 2026-02-20T22:14:22.881 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28041.log 2026-02-20T22:14:22.881 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34460.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.34460.log.gz 2026-02-20T22:14:22.882 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23496.log 2026-02-20T22:14:22.882 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28041.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.28041.log.gz 2026-02-20T22:14:22.882 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28243.log 2026-02-20T22:14:22.883 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.23496.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.23496.log.gz 2026-02-20T22:14:22.883 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35337.log 2026-02-20T22:14:22.884 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28243.log: 82.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28243.log.gz 2026-02-20T22:14:22.884 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47616.log 2026-02-20T22:14:22.885 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35337.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.35337.log.gz 2026-02-20T22:14:22.885 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47471.log 2026-02-20T22:14:22.886 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47616.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47616.log.gz 2026-02-20T22:14:22.886 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44308.log 2026-02-20T22:14:22.888 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47471.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.47471.log.gz 2026-02-20T22:14:22.888 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38234.log 2026-02-20T22:14:22.889 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44308.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44308.log.gz 2026-02-20T22:14:22.889 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41980.log 2026-02-20T22:14:22.890 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38234.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.38234.log.gz 2026-02-20T22:14:22.891 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30214.log 2026-02-20T22:14:22.892 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41980.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41980.log.gz 2026-02-20T22:14:22.892 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26039.log 2026-02-20T22:14:22.892 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30214.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.30214.log.gz 2026-02-20T22:14:22.893 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45729.log 2026-02-20T22:14:22.893 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26039.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26039.log.gz 2026-02-20T22:14:22.894 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41017.log 2026-02-20T22:14:22.895 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45729.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.45729.log.gz 2026-02-20T22:14:22.895 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41154.log 2026-02-20T22:14:22.896 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41017.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41017.log.gz 2026-02-20T22:14:22.896 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35472.log 2026-02-20T22:14:22.898 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41154.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41154.log.gz 2026-02-20T22:14:22.898 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19816.log 2026-02-20T22:14:22.899 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35472.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.35472.log.gz 2026-02-20T22:14:22.899 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46132.log 2026-02-20T22:14:22.900 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19816.log: 56.0% -- replaced with /var/log/ceph/cluster1-client.mirror.19816.log.gz 2026-02-20T22:14:22.900 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40788.log 2026-02-20T22:14:22.901 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46132.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.46132.log.gz 2026-02-20T22:14:22.901 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24158.log 2026-02-20T22:14:22.902 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40788.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.40788.log.gz 2026-02-20T22:14:22.902 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37079.log 2026-02-20T22:14:22.902 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24158.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.24158.log.gz 2026-02-20T22:14:22.903 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50828.log 2026-02-20T22:14:22.903 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37079.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.37079.log.gz 2026-02-20T22:14:22.904 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44513.log 2026-02-20T22:14:22.904 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50828.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50828.log.gz 2026-02-20T22:14:22.904 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23275.log 2026-02-20T22:14:22.905 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44513.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.44513.log.gz 2026-02-20T22:14:22.905 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30910.log 2026-02-20T22:14:22.906 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23275.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.23275.log.gz 2026-02-20T22:14:22.907 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34349.log 2026-02-20T22:14:22.907 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30910.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30910.log.gz 2026-02-20T22:14:22.907 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36048.log 2026-02-20T22:14:22.908 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34349.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.34349.log.gz 2026-02-20T22:14:22.908 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33426.log 2026-02-20T22:14:22.909 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36048.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36048.log.gz 2026-02-20T22:14:22.909 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19112.log 2026-02-20T22:14:22.911 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33426.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.33426.log.gz 2026-02-20T22:14:22.911 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37356.log 2026-02-20T22:14:22.911 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19112.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19112.log.gz 2026-02-20T22:14:22.912 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44100.log 2026-02-20T22:14:22.912 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37356.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37356.log.gz 2026-02-20T22:14:22.913 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27773.log 2026-02-20T22:14:22.914 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44100.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.44100.log.gz 2026-02-20T22:14:22.914 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47575.log 2026-02-20T22:14:22.917 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27773.log: 91.1% -- replaced with /var/log/ceph/cluster2-client.mirror.27773.log.gz 2026-02-20T22:14:22.917 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22095.log 2026-02-20T22:14:22.917 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47575.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47575.log.gz 2026-02-20T22:14:22.918 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33679.log 2026-02-20T22:14:22.925 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22095.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45132.log 2026-02-20T22:14:22.929 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.33679.log: 92.4% 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28221.log.gz 2026-02-20T22:14:22.941 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.33679.log.gz 2026-02-20T22:14:22.941 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43341.log 2026-02-20T22:14:22.943 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45132.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.45132.log.gz 2026-02-20T22:14:22.943 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43501.log 2026-02-20T22:14:22.945 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43341.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43341.log.gz 2026-02-20T22:14:22.945 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35614.log 2026-02-20T22:14:22.946 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43501.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.43501.log.gz 2026-02-20T22:14:22.946 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19467.log 2026-02-20T22:14:22.947 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.35614.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.35614.log.gz 2026-02-20T22:14:22.947 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45893.log 2026-02-20T22:14:22.948 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19467.log: 77.7% -- replaced with /var/log/ceph/cluster1-client.mirror.19467.log.gz 2026-02-20T22:14:22.948 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.44879.log 2026-02-20T22:14:22.949 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45893.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.45893.log.gz 2026-02-20T22:14:22.949 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24295.log 2026-02-20T22:14:22.951 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.44879.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.44879.log.gz 2026-02-20T22:14:22.951 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48620.log 2026-02-20T22:14:22.955 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24295.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.24295.log.gz 2026-02-20T22:14:22.956 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19876.log 2026-02-20T22:14:22.957 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48620.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48620.log.gz 2026-02-20T22:14:22.958 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28391.log 2026-02-20T22:14:22.958 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.19876.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19876.log.gz 2026-02-20T22:14:22.959 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46946.log 2026-02-20T22:14:22.966 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28391.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30969.log 2026-02-20T22:14:22.978 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46946.log: 90.5%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41366.log 2026-02-20T22:14:22.978 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.46946.log.gz 2026-02-20T22:14:22.988 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30969.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31960.log.gz 2026-02-20T22:14:22.988 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36251.log 2026-02-20T22:14:22.990 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41366.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.41366.log.gz 2026-02-20T22:14:22.990 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36301.log 2026-02-20T22:14:22.994 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.36251.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36251.log.gz 2026-02-20T22:14:22.994 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27591.log 2026-02-20T22:14:22.994 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36301.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36301.log.gz 2026-02-20T22:14:22.994 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28549.log.gz 2026-02-20T22:14:22.994 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37892.log 2026-02-20T22:14:22.994 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27591.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27591.log.gz 2026-02-20T22:14:22.995 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22300.log 2026-02-20T22:14:22.995 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46864.log 2026-02-20T22:14:22.995 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37892.log: /var/log/ceph/cluster2-client.mirror.22300.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37892.log.gz 2026-02-20T22:14:22.996 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30194.log 2026-02-20T22:14:22.997 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46864.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.22300.log.gz 2026-02-20T22:14:22.997 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46864.log.gz 2026-02-20T22:14:22.997 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32789.log 2026-02-20T22:14:22.997 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30194.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30194.log.gz 2026-02-20T22:14:22.997 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37793.log 2026-02-20T22:14:22.998 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32789.log: 2.6% -- replaced with /var/log/ceph/cluster2-client.mirror.32789.log.gz 2026-02-20T22:14:22.998 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42592.log 2026-02-20T22:14:22.999 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37793.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.37793.log.gz 2026-02-20T22:14:22.999 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50333.log 2026-02-20T22:14:23.000 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42592.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.42592.log.gz 2026-02-20T22:14:23.000 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47637.log 2026-02-20T22:14:23.000 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50333.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.mirror.50333.log.gz 2026-02-20T22:14:23.001 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28628.log 2026-02-20T22:14:23.001 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.47637.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.47637.log.gz 2026-02-20T22:14:23.002 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48827.log 2026-02-20T22:14:23.010 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28628.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40423.log 2026-02-20T22:14:23.010 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48827.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48827.log.gz 2026-02-20T22:14:23.012 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31086.log.gz 2026-02-20T22:14:23.012 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23081.log 2026-02-20T22:14:23.013 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.40423.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40423.log.gz 2026-02-20T22:14:23.014 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.21323.log 2026-02-20T22:14:23.017 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.23081.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.23081.log.gz 2026-02-20T22:14:23.018 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49138.log 2026-02-20T22:14:23.027 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.1.21323.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30646.log.gz 2026-02-20T22:14:23.028 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41220.log 2026-02-20T22:14:23.029 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49138.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49138.log.gz 2026-02-20T22:14:23.029 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20112.log 2026-02-20T22:14:23.030 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41220.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.41220.log.gz 2026-02-20T22:14:23.031 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31196.log 2026-02-20T22:14:23.031 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.20112.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20112.log.gz 2026-02-20T22:14:23.031 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42030.log 2026-02-20T22:14:23.032 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31196.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31196.log.gz 2026-02-20T22:14:23.032 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52140.log 2026-02-20T22:14:23.032 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42030.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.42030.log.gz 2026-02-20T22:14:23.033 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26830.log 2026-02-20T22:14:23.034 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52140.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.52140.log.gz 2026-02-20T22:14:23.034 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37308.log 2026-02-20T22:14:23.037 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26830.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26830.log.gz 2026-02-20T22:14:23.038 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51737.log 2026-02-20T22:14:23.038 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37308.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37308.log.gz 2026-02-20T22:14:23.038 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50963.log 2026-02-20T22:14:23.039 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.51737.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51737.log.gz 2026-02-20T22:14:23.039 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27571.log 2026-02-20T22:14:23.040 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50963.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50963.log.gz 2026-02-20T22:14:23.040 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54013.log 2026-02-20T22:14:23.041 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27571.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27571.log.gz 2026-02-20T22:14:23.041 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24884.log 2026-02-20T22:14:23.041 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54013.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.54013.log.gz 2026-02-20T22:14:23.050 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37196.log 2026-02-20T22:14:23.066 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24884.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49591.log 2026-02-20T22:14:23.067 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37196.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37196.log.gz 2026-02-20T22:14:23.083 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25805.log 2026-02-20T22:14:23.083 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49591.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49591.log.gz 2026-02-20T22:14:23.098 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26959.log 2026-02-20T22:14:23.114 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25805.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41387.log 2026-02-20T22:14:23.116 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.26959.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32745.log.gz 2026-02-20T22:14:23.117 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.1.21323.log.gz 2026-02-20T22:14:23.117 INFO:teuthology.orchestra.run.trial107.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26959.log.gz 2026-02-20T22:14:23.117 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30097.log 2026-02-20T22:14:23.119 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41387.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41387.log.gz 2026-02-20T22:14:23.119 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34757.log 2026-02-20T22:14:23.120 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30097.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.30097.log.gz 2026-02-20T22:14:23.134 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35683.log 2026-02-20T22:14:23.135 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34757.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.34757.log.gz 2026-02-20T22:14:23.135 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23398.log 2026-02-20T22:14:23.136 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.35683.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35683.log.gz 2026-02-20T22:14:23.137 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.39554.log 2026-02-20T22:14:23.138 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.23398.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.23398.log.gz 2026-02-20T22:14:23.138 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24275.log 2026-02-20T22:14:23.141 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.0.39554.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.39554.log.gz 2026-02-20T22:14:23.141 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50108.log 2026-02-20T22:14:23.145 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.24275.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24275.log.gz 2026-02-20T22:14:23.145 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53695.log 2026-02-20T22:14:23.146 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50108.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50108.log.gz 2026-02-20T22:14:23.146 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46740.log 2026-02-20T22:14:23.147 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53695.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53695.log.gz 2026-02-20T22:14:23.147 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32211.log 2026-02-20T22:14:23.148 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46740.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.46740.log.gz 2026-02-20T22:14:23.148 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34418.log 2026-02-20T22:14:23.149 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32211.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.32211.log.gz 2026-02-20T22:14:23.149 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42147.log 2026-02-20T22:14:23.151 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34418.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.34418.log.gz 2026-02-20T22:14:23.151 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42324.log 2026-02-20T22:14:23.153 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42147.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.42147.log.gz 2026-02-20T22:14:23.153 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29430.log 2026-02-20T22:14:23.154 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42324.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.42324.log.gz 2026-02-20T22:14:23.154 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25783.log 2026-02-20T22:14:23.161 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18574.log 2026-02-20T22:14:23.172 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.25783.log: 90.1% 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.19411.log.gz 2026-02-20T22:14:23.172 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.25783.log.gz 2026-02-20T22:14:23.172 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32387.log 2026-02-20T22:14:23.173 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18574.log: 80.4% -- replaced with /var/log/ceph/cluster2-client.admin.18574.log.gz 2026-02-20T22:14:23.173 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48496.log 2026-02-20T22:14:23.174 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32387.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.32387.log.gz 2026-02-20T22:14:23.174 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28823.log 2026-02-20T22:14:23.176 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48496.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48496.log.gz 2026-02-20T22:14:23.176 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.18784.log 2026-02-20T22:14:23.177 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28823.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28823.log.gz 2026-02-20T22:14:23.178 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38808.log 2026-02-20T22:14:23.178 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.18784.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.18784.log.gz 2026-02-20T22:14:23.178 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27391.log 2026-02-20T22:14:23.179 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38808.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.38808.log.gz 2026-02-20T22:14:23.180 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22073.log 2026-02-20T22:14:23.181 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27391.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.27391.log.gz 2026-02-20T22:14:23.181 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34373.log 2026-02-20T22:14:23.182 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.22073.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.22073.log.gz 2026-02-20T22:14:23.182 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38467.log 2026-02-20T22:14:23.183 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.34373.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.34373.log.gz 2026-02-20T22:14:23.183 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.19782.log 2026-02-20T22:14:23.185 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38467.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38467.log.gz 2026-02-20T22:14:23.185 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20022.log 2026-02-20T22:14:23.187 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.19782.log: 64.8% -- replaced with /var/log/ceph/cluster1-client.mirror.19782.log.gz 2026-02-20T22:14:23.187 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23448.log 2026-02-20T22:14:23.187 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.20022.log: 65.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20022.log.gz 2026-02-20T22:14:23.188 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50266.log 2026-02-20T22:14:23.188 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.23448.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.23448.log.gz 2026-02-20T22:14:23.188 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44465.log 2026-02-20T22:14:23.189 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50266.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50266.log.gz 2026-02-20T22:14:23.189 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41726.log 2026-02-20T22:14:23.189 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44465.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.44465.log.gz 2026-02-20T22:14:23.190 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31726.log 2026-02-20T22:14:23.191 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.41726.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.41726.log.gz 2026-02-20T22:14:23.191 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32435.log 2026-02-20T22:14:23.192 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31726.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.31726.log.gz 2026-02-20T22:14:23.193 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52503.log 2026-02-20T22:14:23.193 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32435.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32435.log.gz 2026-02-20T22:14:23.193 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53582.log 2026-02-20T22:14:23.194 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52503.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52503.log.gz 2026-02-20T22:14:23.195 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39706.log 2026-02-20T22:14:23.195 INFO:teuthology.orchestra.run.trial107.stderr: 94.2%/var/log/ceph/cluster1-client.mirror.53582.log: -- replaced with /var/log/ceph/cluster2-client.mirror.31768.log.gz 2026-02-20T22:14:23.195 INFO:teuthology.orchestra.run.trial107.stderr: 82.1% -- replaced with /var/log/ceph/cluster1-client.mirror.53582.log.gz 2026-02-20T22:14:23.195 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53948.log 2026-02-20T22:14:23.196 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39706.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41660.log 2026-02-20T22:14:23.196 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.53948.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.39706.log.gz 2026-02-20T22:14:23.196 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53948.log.gz 2026-02-20T22:14:23.197 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43573.log 2026-02-20T22:14:23.197 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.41660.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42006.log 2026-02-20T22:14:23.198 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43573.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.43573.log.gz 2026-02-20T22:14:23.198 INFO:teuthology.orchestra.run.trial107.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.41660.log.gz 2026-02-20T22:14:23.198 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38930.log 2026-02-20T22:14:23.199 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24708.log 2026-02-20T22:14:23.199 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42006.log: /var/log/ceph/cluster1-client.mirror.38930.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.42006.log.gz 2026-02-20T22:14:23.199 INFO:teuthology.orchestra.run.trial107.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.38930.log.gz 2026-02-20T22:14:23.199 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38764.log 2026-02-20T22:14:23.200 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.24708.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48848.log 2026-02-20T22:14:23.200 INFO:teuthology.orchestra.run.trial107.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.24708.log.gz 2026-02-20T22:14:23.201 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.38764.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19162.log 2026-02-20T22:14:23.201 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48848.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.38764.log.gz 2026-02-20T22:14:23.201 INFO:teuthology.orchestra.run.trial107.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48848.log.gz 2026-02-20T22:14:23.201 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42189.log 2026-02-20T22:14:23.201 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.admin.19162.log: gzip -5 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19162.log.gz 2026-02-20T22:14:23.202 INFO:teuthology.orchestra.run.trial107.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.25264.log 2026-02-20T22:14:23.202 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28843.log 2026-02-20T22:14:23.203 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.42189.log: /var/log/ceph/cluster2-client.mirror.25264.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42189.log.gz 2026-02-20T22:14:23.203 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34251.log 2026-02-20T22:14:23.204 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28843.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28843.log.gz 2026-02-20T22:14:23.204 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.43174.log 2026-02-20T22:14:23.205 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34251.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.34251.log.gz 2026-02-20T22:14:23.206 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21895.log 2026-02-20T22:14:23.206 INFO:teuthology.orchestra.run.trial107.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25264.log.gz 2026-02-20T22:14:23.206 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.43174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39229.log 2026-02-20T22:14:23.206 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.21895.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.21895.log.gz 2026-02-20T22:14:23.207 INFO:teuthology.orchestra.run.trial107.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.43174.log.gz 2026-02-20T22:14:23.207 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22997.log 2026-02-20T22:14:23.207 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.39229.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.45224.log 2026-02-20T22:14:23.208 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22997.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.22997.log.gz 2026-02-20T22:14:23.208 INFO:teuthology.orchestra.run.trial107.stderr: 92.6%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25284.log 2026-02-20T22:14:23.208 INFO:teuthology.orchestra.run.trial107.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.39229.log.gz 2026-02-20T22:14:23.209 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.45224.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36096.log 2026-02-20T22:14:23.209 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.25284.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.45224.log.gz 2026-02-20T22:14:23.209 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45982.log 2026-02-20T22:14:23.210 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.36096.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.36096.log.gz 2026-02-20T22:14:23.210 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51548.log 2026-02-20T22:14:23.212 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45982.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.45982.log.gz 2026-02-20T22:14:23.212 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26506.log 2026-02-20T22:14:23.212 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.51548.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.25284.log.gz 2026-02-20T22:14:23.213 INFO:teuthology.orchestra.run.trial107.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51548.log.gz 2026-02-20T22:14:23.213 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29272.log 2026-02-20T22:14:23.213 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.26506.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32174.log 2026-02-20T22:14:23.216 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29272.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.26506.log.gz 2026-02-20T22:14:23.217 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48765.log 2026-02-20T22:14:23.217 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.32174.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.32174.log.gz 2026-02-20T22:14:23.217 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22973.log 2026-02-20T22:14:23.218 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48765.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48765.log.gz 2026-02-20T22:14:23.218 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37380.log 2026-02-20T22:14:23.219 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22973.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.22973.log.gz 2026-02-20T22:14:23.219 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.45248.log 2026-02-20T22:14:23.220 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.37380.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.37380.log.gz 2026-02-20T22:14:23.220 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49816.log 2026-02-20T22:14:23.220 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.45248.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.45248.log.gz 2026-02-20T22:14:23.221 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52051.log 2026-02-20T22:14:23.221 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.49816.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49816.log.gz 2026-02-20T22:14:23.222 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22925.log 2026-02-20T22:14:23.222 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.52051.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.52051.log.gz 2026-02-20T22:14:23.223 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49283.log 2026-02-20T22:14:23.223 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.22925.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.22925.log.gz 2026-02-20T22:14:23.223 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40053.log 2026-02-20T22:14:23.224 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.49283.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49283.log.gz 2026-02-20T22:14:23.224 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50041.log 2026-02-20T22:14:23.225 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.40053.log: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.40053.log.gz 2026-02-20T22:14:23.225 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27431.log 2026-02-20T22:14:23.225 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.50041.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50041.log.gz 2026-02-20T22:14:23.226 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38651.log 2026-02-20T22:14:23.226 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.27431.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.27431.log.gz 2026-02-20T22:14:23.227 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42102.log 2026-02-20T22:14:23.227 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.38651.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.38651.log.gz 2026-02-20T22:14:23.228 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29080.log 2026-02-20T22:14:23.228 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.42102.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.42102.log.gz 2026-02-20T22:14:23.229 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28727.log 2026-02-20T22:14:23.229 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.29080.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29080.log.gz 2026-02-20T22:14:23.229 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.46173.log 2026-02-20T22:14:23.230 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28727.log: 80.7% -- replaced with /var/log/ceph/cluster2-client.mirror.28727.log.gz 2026-02-20T22:14:23.230 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.19362.log 2026-02-20T22:14:23.232 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.46173.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.46173.log.gz 2026-02-20T22:14:23.232 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48198.log 2026-02-20T22:14:23.236 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.1.19362.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54155.log.gz 2026-02-20T22:14:23.238 INFO:teuthology.orchestra.run.trial107.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.31552.log.gz 2026-02-20T22:14:23.239 INFO:teuthology.orchestra.run.trial107.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.1.19362.log.gz 2026-02-20T22:14:23.239 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.52369.log 2026-02-20T22:14:23.240 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.48198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.44439.log 2026-02-20T22:14:23.240 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.3.52369.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.48198.log.gz 2026-02-20T22:14:23.240 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.22750.log 2026-02-20T22:14:23.240 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.44439.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37561.log 2026-02-20T22:14:23.241 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.2.22750.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.44439.log.gz 2026-02-20T22:14:23.241 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34504.log 2026-02-20T22:14:23.241 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.37561.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.37561.log.gz 2026-02-20T22:14:23.242 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54954.log 2026-02-20T22:14:23.243 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.34504.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34504.log.gz 2026-02-20T22:14:23.243 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28786.log 2026-02-20T22:14:23.243 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.admin.54954.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.54954.log.gz 2026-02-20T22:14:23.244 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30697.log 2026-02-20T22:14:23.247 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.28786.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.52369.log.gz 2026-02-20T22:14:23.248 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.22750.log.gz 2026-02-20T22:14:23.258 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31298.log 2026-02-20T22:14:23.258 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.30697.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43614.log 2026-02-20T22:14:23.259 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.31298.log: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.31298.log.gz 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.30697.log.gz 2026-02-20T22:14:23.259 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:14:23.259 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52565.log 2026-02-20T22:14:23.259 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.43614.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54200.log 2026-02-20T22:14:23.260 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster2-client.mirror.52565.log: 80.6% -- replaced with /var/log/ceph/cluster2-client.mirror.52565.log.gz 2026-02-20T22:14:23.260 INFO:teuthology.orchestra.run.trial107.stderr:/var/log/ceph/cluster1-client.mirror.54200.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.43614.log.gz 2026-02-20T22:14:23.261 INFO:teuthology.orchestra.run.trial107.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.54200.log.gz 2026-02-20T22:14:23.291 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30859.log.gz 2026-02-20T22:14:23.314 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29826.log.gz 2026-02-20T22:14:23.413 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.30076.log.gz 2026-02-20T22:14:23.432 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28628.log.gz 2026-02-20T22:14:23.484 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.31255.log.gz 2026-02-20T22:14:23.499 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29997.log.gz 2026-02-20T22:14:23.514 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28786.log.gz 2026-02-20T22:14:23.526 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53903.log.gz 2026-02-20T22:14:23.545 INFO:teuthology.orchestra.run.trial107.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.30969.log.gz 2026-02-20T22:14:23.552 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.25805.log.gz 2026-02-20T22:14:23.552 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32574.log.gz 2026-02-20T22:14:23.574 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52966.log.gz 2026-02-20T22:14:23.578 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54407.log.gz 2026-02-20T22:14:23.611 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29588.log.gz 2026-02-20T22:14:23.657 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22095.log.gz 2026-02-20T22:14:23.696 INFO:teuthology.orchestra.run.trial107.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29430.log.gz 2026-02-20T22:14:23.734 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.29272.log.gz 2026-02-20T22:14:23.801 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.24884.log.gz 2026-02-20T22:14:23.837 INFO:teuthology.orchestra.run.trial107.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28391.log.gz 2026-02-20T22:14:24.053 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.0 has been restored 2026-02-20T22:14:26.369 INFO:teuthology.orchestra.run.trial032.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-20T22:14:26.556 INFO:teuthology.orchestra.run.trial032.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-20T22:14:26.744 INFO:teuthology.orchestra.run.trial032.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-20T22:14:26.746 INFO:teuthology.orchestra.run.trial032.stderr: 2026-02-20T22:14:26.746 INFO:teuthology.orchestra.run.trial032.stderr:real 0m5.573s 2026-02-20T22:14:26.746 INFO:teuthology.orchestra.run.trial032.stderr:user 0m16.206s 2026-02-20T22:14:26.747 INFO:teuthology.orchestra.run.trial032.stderr:sys 0m0.784s 2026-02-20T22:15:46.979 INFO:teuthology.orchestra.run.trial107.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-20T22:15:49.201 INFO:teuthology.orchestra.run.trial107.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-20T22:16:13.331 INFO:teuthology.orchestra.run.trial107.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-20T22:16:13.331 INFO:teuthology.orchestra.run.trial107.stderr: 2026-02-20T22:16:13.332 INFO:teuthology.orchestra.run.trial107.stderr:real 1m52.152s 2026-02-20T22:16:13.332 INFO:teuthology.orchestra.run.trial107.stderr:user 5m17.748s 2026-02-20T22:16:13.332 INFO:teuthology.orchestra.run.trial107.stderr:sys 0m15.679s 2026-02-20T22:16:13.333 INFO:tasks.ceph:Archiving logs... 2026-02-20T22:16:13.333 DEBUG:teuthology.misc:Transferring archived files from trial032:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486/remote/trial032/log 2026-02-20T22:16:13.335 DEBUG:teuthology.orchestra.run.trial032:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-20T22:16:14.579 DEBUG:teuthology.misc:Transferring archived files from trial107:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486/remote/trial107/log 2026-02-20T22:16:14.587 DEBUG:teuthology.orchestra.run.trial107:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-20T22:16:38.110 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-20T22:16:38.120 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-20T22:16:38.120 DEBUG:teuthology.orchestra.run.trial032:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-20T22:16:38.122 DEBUG:teuthology.orchestra.run.trial107:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-20T22:16:38.263 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-20T22:16:38.277 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-20T22:16:38.277 DEBUG:teuthology.orchestra.run.trial032:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T22:16:38.278 DEBUG:teuthology.orchestra.run.trial107:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T22:16:38.333 INFO:teuthology.orchestra.run.trial032.stdout: remote refid st t when poll reach delay offset jitter 2026-02-20T22:16:38.333 INFO:teuthology.orchestra.run.trial032.stdout:============================================================================== 2026-02-20T22:16:38.333 INFO:teuthology.orchestra.run.trial032.stdout:*frumentum-zeus. 209.180.247.49 2 u 102 128 377 29.692 +3.187 0.780 2026-02-20T22:16:38.333 INFO:teuthology.orchestra.run.trial032.stdout:+45.83.234.123 ( 10.1.105.4 2 u 97 128 377 81.451 +0.574 2.029 2026-02-20T22:16:38.333 INFO:teuthology.orchestra.run.trial032.stdout:-23.157.160.168 129.6.15.28 2 u 114 128 377 3.242 +0.887 1.969 2026-02-20T22:16:38.333 INFO:teuthology.orchestra.run.trial032.stdout:+falkor.smatwebd 192.155.88.241 3 u 105 128 377 8.769 +1.515 0.678 2026-02-20T22:16:38.689 INFO:teuthology.orchestra.run.trial107.stdout: remote refid st t when poll reach delay offset jitter 2026-02-20T22:16:38.690 INFO:teuthology.orchestra.run.trial107.stdout:============================================================================== 2026-02-20T22:16:38.690 INFO:teuthology.orchestra.run.trial107.stdout:+triton.ellipse. 50.43.156.178 2 u 35 128 377 31.228 -0.105 0.642 2026-02-20T22:16:38.690 INFO:teuthology.orchestra.run.trial107.stdout:*12.193.130.137 .PPS. 1 u 40 128 377 39.410 -2.025 2.163 2026-02-20T22:16:38.690 INFO:teuthology.orchestra.run.trial107.stdout:+s2-b.time.mci1. 42.20.202.230 2 u 58 128 377 29.810 -0.691 0.360 2026-02-20T22:16:38.690 INFO:teuthology.orchestra.run.trial107.stdout:-ntp.maxhost.io 132.163.97.4 2 u 107 128 133 69.713 -0.282 2.931 2026-02-20T22:16:38.691 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-20T22:16:38.707 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-20T22:16:38.707 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-20T22:16:38.720 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-20T22:16:38.733 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-20T22:16:38.746 INFO:teuthology.task.internal:Duration was 2573.972510 seconds 2026-02-20T22:16:38.746 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-20T22:16:38.760 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-20T22:16:38.760 DEBUG:teuthology.orchestra.run.trial032:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-20T22:16:38.761 DEBUG:teuthology.orchestra.run.trial107:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-20T22:16:38.785 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-20T22:16:38.785 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial032.front.sepia.ceph.com 2026-02-20T22:16:38.785 DEBUG:teuthology.orchestra.run.trial032:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-20T22:16:38.832 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial107.front.sepia.ceph.com 2026-02-20T22:16:38.832 DEBUG:teuthology.orchestra.run.trial107:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-20T22:16:38.838 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-20T22:16:38.838 DEBUG:teuthology.orchestra.run.trial032:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T22:16:38.875 DEBUG:teuthology.orchestra.run.trial107:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T22:16:39.100 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-20T22:16:39.100 DEBUG:teuthology.orchestra.run.trial032:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T22:16:39.101 DEBUG:teuthology.orchestra.run.trial107:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T22:16:39.109 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T22:16:39.109 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T22:16:39.109 INFO:teuthology.orchestra.run.trial032.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-20T22:16:39.109 INFO:teuthology.orchestra.run.trial032.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T22:16:39.110 INFO:teuthology.orchestra.run.trial032.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-20T22:16:39.110 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T22:16:39.110 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T22:16:39.110 INFO:teuthology.orchestra.run.trial107.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-20T22:16:39.110 INFO:teuthology.orchestra.run.trial107.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T22:16:39.111 INFO:teuthology.orchestra.run.trial107.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-20T22:16:39.177 INFO:teuthology.orchestra.run.trial032.stderr: 81.4% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-20T22:16:39.180 INFO:teuthology.orchestra.run.trial107.stderr: 81.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-20T22:16:39.182 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-20T22:16:39.197 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-20T22:16:39.198 DEBUG:teuthology.orchestra.run.trial032:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-20T22:16:39.230 DEBUG:teuthology.orchestra.run.trial107:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-20T22:16:39.238 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-20T22:16:39.253 DEBUG:teuthology.orchestra.run.trial032:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-20T22:16:39.271 DEBUG:teuthology.orchestra.run.trial107:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-20T22:16:39.278 INFO:teuthology.orchestra.run.trial032.stdout:kernel.core_pattern = core 2026-02-20T22:16:39.288 INFO:teuthology.orchestra.run.trial107.stdout:kernel.core_pattern = core 2026-02-20T22:16:39.298 DEBUG:teuthology.orchestra.run.trial032:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-20T22:16:39.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T22:16:39.334 DEBUG:teuthology.orchestra.run.trial107:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-20T22:16:39.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T22:16:39.344 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-20T22:16:39.360 INFO:teuthology.task.internal:Transferring archived files... 2026-02-20T22:16:39.360 DEBUG:teuthology.misc:Transferring archived files from trial032:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486/remote/trial032 2026-02-20T22:16:39.360 DEBUG:teuthology.orchestra.run.trial032:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-20T22:16:39.404 DEBUG:teuthology.misc:Transferring archived files from trial107:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61486/remote/trial107 2026-02-20T22:16:39.404 DEBUG:teuthology.orchestra.run.trial107:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-20T22:16:39.432 INFO:teuthology.task.internal:Removing archive directory... 2026-02-20T22:16:39.432 DEBUG:teuthology.orchestra.run.trial032:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-20T22:16:39.433 DEBUG:teuthology.orchestra.run.trial107:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-20T22:16:39.465 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-20T22:16:39.480 INFO:teuthology.task.internal:Not uploading archives. 2026-02-20T22:16:39.480 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-20T22:16:39.497 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-20T22:16:39.497 DEBUG:teuthology.orchestra.run.trial032:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-20T22:16:39.498 DEBUG:teuthology.orchestra.run.trial107:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-20T22:16:39.501 INFO:teuthology.orchestra.run.trial032.stdout: 1179659 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 20 22:16 /home/ubuntu/cephtest 2026-02-20T22:16:39.508 INFO:teuthology.orchestra.run.trial107.stdout: 1179659 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 20 22:16 /home/ubuntu/cephtest 2026-02-20T22:16:39.509 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-20T22:16:39.522 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-20T22:16:39.559 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-comp-snappy policy/none rbd-mirror/four-per-cluster supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-snapshot-stress-workunit-fast-diff} duration: 2573.9725103378296 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-20T22:16:39.559 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-20T22:16:39.658 INFO:teuthology.run:pass